gen.go

Documentation: golang.org/x/text/internal/gen

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package gen contains common code for the various code generation tools in the
     6  // text repository. Its usage ensures consistency between tools.
     7  //
     8  // This package defines command line flags that are common to most generation
     9  // tools. The flags allow for specifying specific Unicode and CLDR versions
    10  // in the public Unicode data repository (https://www.unicode.org/Public).
    11  //
    12  // A local Unicode data mirror can be set through the flag -local or the
    13  // environment variable UNICODE_DIR. The former takes precedence. The local
    14  // directory should follow the same structure as the public repository.
    15  //
    16  // IANA data can also optionally be mirrored by putting it in the iana directory
    17  // rooted at the top of the local mirror. Beware, though, that IANA data is not
    18  // versioned. So it is up to the developer to use the right version.
    19  package gen // import "golang.org/x/text/internal/gen"
    20  
    21  import (
    22  	"bytes"
    23  	"flag"
    24  	"fmt"
    25  	"go/build"
    26  	"go/format"
    27  	"io"
    28  	"log"
    29  	"net/http"
    30  	"os"
    31  	"path"
    32  	"path/filepath"
    33  	"regexp"
    34  	"strings"
    35  	"sync"
    36  	"unicode"
    37  
    38  	"golang.org/x/text/unicode/cldr"
    39  )
    40  
    41  var (
    42  	url = flag.String("url",
    43  		"https://www.unicode.org/Public",
    44  		"URL of Unicode database directory")
    45  	iana = flag.String("iana",
    46  		"http://www.iana.org",
    47  		"URL of the IANA repository")
    48  	unicodeVersion = flag.String("unicode",
    49  		getEnv("UNICODE_VERSION", unicode.Version),
    50  		"unicode version to use")
    51  	cldrVersion = flag.String("cldr",
    52  		getEnv("CLDR_VERSION", cldr.Version),
    53  		"cldr version to use")
    54  )
    55  
    56  func getEnv(name, def string) string {
    57  	if v := os.Getenv(name); v != "" {
    58  		return v
    59  	}
    60  	return def
    61  }
    62  
    63  // Init performs common initialization for a gen command. It parses the flags
    64  // and sets up the standard logging parameters.
    65  func Init() {
    66  	log.SetPrefix("")
    67  	log.SetFlags(log.Lshortfile)
    68  	flag.Parse()
    69  }
    70  
    71  const header = `// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
    72  
    73  `
    74  
    75  // UnicodeVersion reports the requested Unicode version.
    76  func UnicodeVersion() string {
    77  	return *unicodeVersion
    78  }
    79  
    80  // CLDRVersion reports the requested CLDR version.
    81  func CLDRVersion() string {
    82  	return *cldrVersion
    83  }
    84  
    85  var tags = []struct{ version, buildTags string }{
    86  	{"9.0.0", "!go1.10"},
    87  	{"10.0.0", "go1.10,!go1.13"},
    88  	{"11.0.0", "go1.13,!go1.14"},
    89  	{"12.0.0", "go1.14,!go1.16"},
    90  	{"13.0.0", "go1.16,!go1.21"},
    91  	{"15.0.0", "go1.21"},
    92  }
    93  
    94  // buildTags reports the build tags used for the current Unicode version.
    95  func buildTags() string {
    96  	v := UnicodeVersion()
    97  	for _, e := range tags {
    98  		if e.version == v {
    99  			return e.buildTags
   100  		}
   101  	}
   102  	log.Fatalf("Unknown build tags for Unicode version %q.", v)
   103  	return ""
   104  }
   105  
   106  // IsLocal reports whether data files are available locally.
   107  func IsLocal() bool {
   108  	dir, err := localReadmeFile()
   109  	if err != nil {
   110  		return false
   111  	}
   112  	if _, err = os.Stat(dir); err != nil {
   113  		return false
   114  	}
   115  	return true
   116  }
   117  
   118  // OpenUCDFile opens the requested UCD file. The file is specified relative to
   119  // the public Unicode root directory. It will call log.Fatal if there are any
   120  // errors.
   121  func OpenUCDFile(file string) io.ReadCloser {
   122  	return openUnicode(path.Join(*unicodeVersion, "ucd", file))
   123  }
   124  
   125  // OpenCLDRCoreZip opens the CLDR core zip file. It will call log.Fatal if there
   126  // are any errors.
   127  func OpenCLDRCoreZip() io.ReadCloser {
   128  	return OpenUnicodeFile("cldr", *cldrVersion, "core.zip")
   129  }
   130  
   131  // OpenUnicodeFile opens the requested file of the requested category from the
   132  // root of the Unicode data archive. The file is specified relative to the
   133  // public Unicode root directory. If version is "", it will use the default
   134  // Unicode version. It will call log.Fatal if there are any errors.
   135  func OpenUnicodeFile(category, version, file string) io.ReadCloser {
   136  	if version == "" {
   137  		version = UnicodeVersion()
   138  	}
   139  	return openUnicode(path.Join(category, version, file))
   140  }
   141  
   142  // OpenIANAFile opens the requested IANA file. The file is specified relative
   143  // to the IANA root, which is typically either http://www.iana.org or the
   144  // iana directory in the local mirror. It will call log.Fatal if there are any
   145  // errors.
   146  func OpenIANAFile(path string) io.ReadCloser {
   147  	return Open(*iana, "iana", path)
   148  }
   149  
   150  var (
   151  	dirMutex sync.Mutex
   152  	localDir string
   153  )
   154  
   155  const permissions = 0755
   156  
   157  func localReadmeFile() (string, error) {
   158  	p, err := build.Import("golang.org/x/text", "", build.FindOnly)
   159  	if err != nil {
   160  		return "", fmt.Errorf("Could not locate package: %v", err)
   161  	}
   162  	return filepath.Join(p.Dir, "DATA", "README"), nil
   163  }
   164  
   165  func getLocalDir() string {
   166  	dirMutex.Lock()
   167  	defer dirMutex.Unlock()
   168  
   169  	readme, err := localReadmeFile()
   170  	if err != nil {
   171  		log.Fatal(err)
   172  	}
   173  	dir := filepath.Dir(readme)
   174  	if _, err := os.Stat(readme); err != nil {
   175  		if err := os.MkdirAll(dir, permissions); err != nil {
   176  			log.Fatalf("Could not create directory: %v", err)
   177  		}
   178  		os.WriteFile(readme, []byte(readmeTxt), permissions)
   179  	}
   180  	return dir
   181  }
   182  
   183  const readmeTxt = `Generated by golang.org/x/text/internal/gen. DO NOT EDIT.
   184  
   185  This directory contains downloaded files used to generate the various tables
   186  in the golang.org/x/text subrepo.
   187  
   188  Note that the language subtag repo (iana/assignments/language-subtag-registry)
   189  and all other times in the iana subdirectory are not versioned and will need
   190  to be periodically manually updated. The easiest way to do this is to remove
   191  the entire iana directory. This is mostly of concern when updating the language
   192  package.
   193  `
   194  
   195  // Open opens subdir/path if a local directory is specified and the file exists,
   196  // where subdir is a directory relative to the local root, or fetches it from
   197  // urlRoot/path otherwise. It will call log.Fatal if there are any errors.
   198  func Open(urlRoot, subdir, path string) io.ReadCloser {
   199  	file := filepath.Join(getLocalDir(), subdir, filepath.FromSlash(path))
   200  	return open(file, urlRoot, path)
   201  }
   202  
   203  func openUnicode(path string) io.ReadCloser {
   204  	file := filepath.Join(getLocalDir(), filepath.FromSlash(path))
   205  	return open(file, *url, path)
   206  }
   207  
   208  // TODO: automatically periodically update non-versioned files.
   209  
   210  func open(file, urlRoot, path string) io.ReadCloser {
   211  	if f, err := os.Open(file); err == nil {
   212  		return f
   213  	}
   214  	r := get(urlRoot, path)
   215  	defer r.Close()
   216  	b, err := io.ReadAll(r)
   217  	if err != nil {
   218  		log.Fatalf("Could not download file: %v", err)
   219  	}
   220  	os.MkdirAll(filepath.Dir(file), permissions)
   221  	if err := os.WriteFile(file, b, permissions); err != nil {
   222  		log.Fatalf("Could not create file: %v", err)
   223  	}
   224  	return io.NopCloser(bytes.NewReader(b))
   225  }
   226  
   227  func get(root, path string) io.ReadCloser {
   228  	url := root + "/" + path
   229  	fmt.Printf("Fetching %s...", url)
   230  	defer fmt.Println(" done.")
   231  	resp, err := http.Get(url)
   232  	if err != nil {
   233  		log.Fatalf("HTTP GET: %v", err)
   234  	}
   235  	if resp.StatusCode != 200 {
   236  		log.Fatalf("Bad GET status for %q: %q", url, resp.Status)
   237  	}
   238  	return resp.Body
   239  }
   240  
   241  // TODO: use Write*Version in all applicable packages.
   242  
   243  // WriteUnicodeVersion writes a constant for the Unicode version from which the
   244  // tables are generated.
   245  func WriteUnicodeVersion(w io.Writer) {
   246  	fmt.Fprintf(w, "// UnicodeVersion is the Unicode version from which the tables in this package are derived.\n")
   247  	fmt.Fprintf(w, "const UnicodeVersion = %q\n\n", UnicodeVersion())
   248  }
   249  
   250  // WriteCLDRVersion writes a constant for the CLDR version from which the
   251  // tables are generated.
   252  func WriteCLDRVersion(w io.Writer) {
   253  	fmt.Fprintf(w, "// CLDRVersion is the CLDR version from which the tables in this package are derived.\n")
   254  	fmt.Fprintf(w, "const CLDRVersion = %q\n\n", CLDRVersion())
   255  }
   256  
   257  // WriteGoFile prepends a standard file comment and package statement to the
   258  // given bytes, applies gofmt, and writes them to a file with the given name.
   259  // It will call log.Fatal if there are any errors.
   260  func WriteGoFile(filename, pkg string, b []byte) {
   261  	w, err := os.Create(filename)
   262  	if err != nil {
   263  		log.Fatalf("Could not create file %s: %v", filename, err)
   264  	}
   265  	defer w.Close()
   266  	if _, err = WriteGo(w, pkg, "", b); err != nil {
   267  		log.Fatalf("Error writing file %s: %v", filename, err)
   268  	}
   269  }
   270  
   271  func fileToPattern(filename string) string {
   272  	suffix := ".go"
   273  	if strings.HasSuffix(filename, "_test.go") {
   274  		suffix = "_test.go"
   275  	}
   276  	prefix := filename[:len(filename)-len(suffix)]
   277  	return fmt.Sprint(prefix, "%s", suffix)
   278  }
   279  
   280  // tagLines returns the //go:build lines to add to the file.
   281  func tagLines(tags string) string {
   282  	return "//go:build " + strings.ReplaceAll(tags, ",", " && ") + "\n"
   283  }
   284  
   285  func updateBuildTags(pattern string) {
   286  	for _, t := range tags {
   287  		oldFile := fmt.Sprintf(pattern, t.version)
   288  		b, err := os.ReadFile(oldFile)
   289  		if err != nil {
   290  			continue
   291  		}
   292  		b = regexp.MustCompile(`//go:build.*\n`).ReplaceAll(b, []byte(tagLines(t.buildTags)))
   293  		err = os.WriteFile(oldFile, b, 0644)
   294  		if err != nil {
   295  			log.Fatal(err)
   296  		}
   297  	}
   298  }
   299  
   300  // WriteVersionedGoFile prepends a standard file comment, adds build tags to
   301  // version the file for the current Unicode version, and package statement to
   302  // the given bytes, applies gofmt, and writes them to a file with the given
   303  // name. It will call log.Fatal if there are any errors.
   304  func WriteVersionedGoFile(filename, pkg string, b []byte) {
   305  	pattern := fileToPattern(filename)
   306  	updateBuildTags(pattern)
   307  	filename = fmt.Sprintf(pattern, UnicodeVersion())
   308  
   309  	w, err := os.Create(filename)
   310  	if err != nil {
   311  		log.Fatalf("Could not create file %s: %v", filename, err)
   312  	}
   313  	defer w.Close()
   314  	if _, err = WriteGo(w, pkg, buildTags(), b); err != nil {
   315  		log.Fatalf("Error writing file %s: %v", filename, err)
   316  	}
   317  }
   318  
   319  // WriteGo prepends a standard file comment and package statement to the given
   320  // bytes, applies gofmt, and writes them to w.
   321  func WriteGo(w io.Writer, pkg, tags string, b []byte) (n int, err error) {
   322  	src := []byte(header)
   323  	if tags != "" {
   324  		src = append(src, tagLines(tags)...)
   325  		src = append(src, '\n')
   326  	}
   327  	src = append(src, fmt.Sprintf("package %s\n\n", pkg)...)
   328  	src = append(src, b...)
   329  	formatted, err := format.Source(src)
   330  	if err != nil {
   331  		// Print the generated code even in case of an error so that the
   332  		// returned error can be meaningfully interpreted.
   333  		n, _ = w.Write(src)
   334  		return n, err
   335  	}
   336  	return w.Write(formatted)
   337  }
   338  
   339  // Repackage rewrites a Go file from belonging to package main to belonging to
   340  // the given package.
   341  func Repackage(inFile, outFile, pkg string) {
   342  	src, err := os.ReadFile(inFile)
   343  	if err != nil {
   344  		log.Fatalf("reading %s: %v", inFile, err)
   345  	}
   346  	const toDelete = "package main\n\n"
   347  	i := bytes.Index(src, []byte(toDelete))
   348  	if i < 0 {
   349  		log.Fatalf("Could not find %q in %s.", toDelete, inFile)
   350  	}
   351  	w := &bytes.Buffer{}
   352  	w.Write(src[i+len(toDelete):])
   353  	WriteGoFile(outFile, pkg, w.Bytes())
   354  }
   355
View as plain text