generate.go

Documentation: golang.org/x/text/message/pipeline

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package pipeline
     6  
     7  import (
     8  	"fmt"
     9  	"go/build"
    10  	"io"
    11  	"os"
    12  	"path/filepath"
    13  	"regexp"
    14  	"sort"
    15  	"strings"
    16  	"text/template"
    17  
    18  	"golang.org/x/text/collate"
    19  	"golang.org/x/text/feature/plural"
    20  	"golang.org/x/text/internal"
    21  	"golang.org/x/text/internal/catmsg"
    22  	"golang.org/x/text/internal/gen"
    23  	"golang.org/x/text/language"
    24  	"golang.org/x/tools/go/loader"
    25  )
    26  
    27  var transRe = regexp.MustCompile(`messages\.(.*)\.json`)
    28  
    29  // Generate writes a Go file that defines a Catalog with translated messages.
    30  // Translations are retrieved from s.Messages, not s.Translations, so it
    31  // is assumed Merge has been called.
    32  func (s *State) Generate() error {
    33  	path := s.Config.GenPackage
    34  	if path == "" {
    35  		path = "."
    36  	}
    37  	isDir := path[0] == '.'
    38  	prog, err := loadPackages(&loader.Config{}, []string{path})
    39  	if err != nil {
    40  		return wrap(err, "could not load package")
    41  	}
    42  	pkgs := prog.InitialPackages()
    43  	if len(pkgs) != 1 {
    44  		return errorf("more than one package selected: %v", pkgs)
    45  	}
    46  	pkg := pkgs[0].Pkg.Name()
    47  
    48  	cw, err := s.generate()
    49  	if err != nil {
    50  		return err
    51  	}
    52  	if !isDir {
    53  		gopath := filepath.SplitList(build.Default.GOPATH)[0]
    54  		path = filepath.Join(gopath, "src", filepath.FromSlash(pkgs[0].Pkg.Path()))
    55  	}
    56  	if len(s.Config.GenFile) == 0 {
    57  		cw.WriteGo(os.Stdout, pkg, "")
    58  		return nil
    59  	}
    60  	if filepath.IsAbs(s.Config.GenFile) {
    61  		path = s.Config.GenFile
    62  	} else {
    63  		path = filepath.Join(path, s.Config.GenFile)
    64  	}
    65  	cw.WriteGoFile(path, pkg) // TODO: WriteGoFile should return error.
    66  	return err
    67  }
    68  
    69  // WriteGen writes a Go file with the given package name to w that defines a
    70  // Catalog with translated messages. Translations are retrieved from s.Messages,
    71  // not s.Translations, so it is assumed Merge has been called.
    72  func (s *State) WriteGen(w io.Writer, pkg string) error {
    73  	cw, err := s.generate()
    74  	if err != nil {
    75  		return err
    76  	}
    77  	_, err = cw.WriteGo(w, pkg, "")
    78  	return err
    79  }
    80  
    81  // Generate is deprecated; use (*State).Generate().
    82  func Generate(w io.Writer, pkg string, extracted *Messages, trans ...Messages) (n int, err error) {
    83  	s := State{
    84  		Extracted:    *extracted,
    85  		Translations: trans,
    86  	}
    87  	cw, err := s.generate()
    88  	if err != nil {
    89  		return 0, err
    90  	}
    91  	return cw.WriteGo(w, pkg, "")
    92  }
    93  
    94  func (s *State) generate() (*gen.CodeWriter, error) {
    95  	// Build up index of translations and original messages.
    96  	translations := map[language.Tag]map[string]Message{}
    97  	languages := []language.Tag{}
    98  	usedKeys := map[string]int{}
    99  
   100  	for _, loc := range s.Messages {
   101  		tag := loc.Language
   102  		if _, ok := translations[tag]; !ok {
   103  			translations[tag] = map[string]Message{}
   104  			languages = append(languages, tag)
   105  		}
   106  		for _, m := range loc.Messages {
   107  			if !m.Translation.IsEmpty() {
   108  				for _, id := range m.ID {
   109  					if _, ok := translations[tag][id]; ok {
   110  						warnf("Duplicate translation in locale %q for message %q", tag, id)
   111  					}
   112  					translations[tag][id] = m
   113  				}
   114  			}
   115  		}
   116  	}
   117  
   118  	// Verify completeness and register keys.
   119  	internal.SortTags(languages)
   120  
   121  	langVars := []string{}
   122  	for _, tag := range languages {
   123  		langVars = append(langVars, strings.Replace(tag.String(), "-", "_", -1))
   124  		dict := translations[tag]
   125  		for _, msg := range s.Extracted.Messages {
   126  			for _, id := range msg.ID {
   127  				if trans, ok := dict[id]; ok && !trans.Translation.IsEmpty() {
   128  					if _, ok := usedKeys[msg.Key]; !ok {
   129  						usedKeys[msg.Key] = len(usedKeys)
   130  					}
   131  					break
   132  				}
   133  				// TODO: log missing entry.
   134  				warnf("%s: Missing entry for %q.", tag, id)
   135  			}
   136  		}
   137  	}
   138  
   139  	cw := gen.NewCodeWriter()
   140  
   141  	x := &struct {
   142  		Fallback  language.Tag
   143  		Languages []string
   144  	}{
   145  		Fallback:  s.Extracted.Language,
   146  		Languages: langVars,
   147  	}
   148  
   149  	if err := lookup.Execute(cw, x); err != nil {
   150  		return nil, wrap(err, "error")
   151  	}
   152  
   153  	keyToIndex := []string{}
   154  	for k := range usedKeys {
   155  		keyToIndex = append(keyToIndex, k)
   156  	}
   157  	sort.Strings(keyToIndex)
   158  	fmt.Fprint(cw, "var messageKeyToIndex = map[string]int{\n")
   159  	for _, k := range keyToIndex {
   160  		fmt.Fprintf(cw, "%q: %d,\n", k, usedKeys[k])
   161  	}
   162  	fmt.Fprint(cw, "}\n\n")
   163  
   164  	for i, tag := range languages {
   165  		dict := translations[tag]
   166  		a := make([]string, len(usedKeys))
   167  		for _, msg := range s.Extracted.Messages {
   168  			for _, id := range msg.ID {
   169  				if trans, ok := dict[id]; ok && !trans.Translation.IsEmpty() {
   170  					m, err := assemble(&msg, &trans.Translation)
   171  					if err != nil {
   172  						return nil, wrap(err, "error")
   173  					}
   174  					_, leadWS, trailWS := trimWS(msg.Key)
   175  					if leadWS != "" || trailWS != "" {
   176  						m = catmsg.Affix{
   177  							Message: m,
   178  							Prefix:  leadWS,
   179  							Suffix:  trailWS,
   180  						}
   181  					}
   182  					// TODO: support macros.
   183  					data, err := catmsg.Compile(tag, nil, m)
   184  					if err != nil {
   185  						return nil, wrap(err, "error")
   186  					}
   187  					key := usedKeys[msg.Key]
   188  					if d := a[key]; d != "" && d != data {
   189  						warnf("Duplicate non-consistent translation for key %q, picking the one for message %q", msg.Key, id)
   190  					}
   191  					a[key] = string(data)
   192  					break
   193  				}
   194  			}
   195  		}
   196  		index := []uint32{0}
   197  		p := 0
   198  		for _, s := range a {
   199  			p += len(s)
   200  			index = append(index, uint32(p))
   201  		}
   202  
   203  		cw.WriteVar(langVars[i]+"Index", index)
   204  		cw.WriteConst(langVars[i]+"Data", strings.Join(a, ""))
   205  	}
   206  	return cw, nil
   207  }
   208  
   209  func assemble(m *Message, t *Text) (msg catmsg.Message, err error) {
   210  	keys := []string{}
   211  	for k := range t.Var {
   212  		keys = append(keys, k)
   213  	}
   214  	sort.Strings(keys)
   215  	var a []catmsg.Message
   216  	for _, k := range keys {
   217  		t := t.Var[k]
   218  		m, err := assemble(m, &t)
   219  		if err != nil {
   220  			return nil, err
   221  		}
   222  		a = append(a, &catmsg.Var{Name: k, Message: m})
   223  	}
   224  	if t.Select != nil {
   225  		s, err := assembleSelect(m, t.Select)
   226  		if err != nil {
   227  			return nil, err
   228  		}
   229  		a = append(a, s)
   230  	}
   231  	if t.Msg != "" {
   232  		sub, err := m.Substitute(t.Msg)
   233  		if err != nil {
   234  			return nil, err
   235  		}
   236  		a = append(a, catmsg.String(sub))
   237  	}
   238  	switch len(a) {
   239  	case 0:
   240  		return nil, errorf("generate: empty message")
   241  	case 1:
   242  		return a[0], nil
   243  	default:
   244  		return catmsg.FirstOf(a), nil
   245  
   246  	}
   247  }
   248  
   249  func assembleSelect(m *Message, s *Select) (msg catmsg.Message, err error) {
   250  	cases := []string{}
   251  	for c := range s.Cases {
   252  		cases = append(cases, c)
   253  	}
   254  	sortCases(cases)
   255  
   256  	caseMsg := []interface{}{}
   257  	for _, c := range cases {
   258  		cm := s.Cases[c]
   259  		m, err := assemble(m, &cm)
   260  		if err != nil {
   261  			return nil, err
   262  		}
   263  		caseMsg = append(caseMsg, c, m)
   264  	}
   265  
   266  	ph := m.Placeholder(s.Arg)
   267  
   268  	switch s.Feature {
   269  	case "plural":
   270  		// TODO: only printf-style selects are supported as of yet.
   271  		return plural.Selectf(ph.ArgNum, ph.String, caseMsg...), nil
   272  	}
   273  	return nil, errorf("unknown feature type %q", s.Feature)
   274  }
   275  
   276  func sortCases(cases []string) {
   277  	// TODO: implement full interface.
   278  	sort.Slice(cases, func(i, j int) bool {
   279  		switch {
   280  		case cases[i] != "other" && cases[j] == "other":
   281  			return true
   282  		case cases[i] == "other" && cases[j] != "other":
   283  			return false
   284  		}
   285  		// the following code relies on '<' < '=' < any letter.
   286  		return cmpNumeric(cases[i], cases[j]) == -1
   287  	})
   288  }
   289  
   290  var cmpNumeric = collate.New(language.Und, collate.Numeric).CompareString
   291  
   292  var lookup = template.Must(template.New("gen").Parse(`
   293  import (
   294  	"golang.org/x/text/language"
   295  	"golang.org/x/text/message"
   296  	"golang.org/x/text/message/catalog"
   297  )
   298  
   299  type dictionary struct {
   300  	index []uint32
   301  	data  string
   302  }
   303  
   304  func (d *dictionary) Lookup(key string) (data string, ok bool) {
   305  	p, ok := messageKeyToIndex[key]
   306  	if !ok {
   307  		return "", false
   308  	}
   309  	start, end := d.index[p], d.index[p+1]
   310  	if start == end {
   311  		return "", false
   312  	}
   313  	return d.data[start:end], true
   314  }
   315  
   316  func init() {
   317  	dict := map[string]catalog.Dictionary{
   318  		{{range .Languages}}"{{.}}": &dictionary{index: {{.}}Index, data: {{.}}Data },
   319  		{{end}}
   320  	}
   321  	fallback := language.MustParse("{{.Fallback}}")
   322  	cat, err := catalog.NewFromMap(dict, catalog.Fallback(fallback))
   323  	if err != nil {
   324  		panic(err)
   325  	}
   326  	message.DefaultCatalog = cat
   327  }
   328  
   329  `))
   330
View as plain text