main.go

Documentation: google.golang.org/protobuf/internal/cmd/generate-protos

     1  // Copyright 2019 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:generate go run -tags protolegacy . -execute
     6  
     7  package main
     8  
     9  import (
    10  	"bytes"
    11  	"flag"
    12  	"fmt"
    13  	"go/format"
    14  	"io/ioutil"
    15  	"os"
    16  	"os/exec"
    17  	"path"
    18  	"path/filepath"
    19  	"regexp"
    20  	"sort"
    21  	"strconv"
    22  	"strings"
    23  
    24  	gengo "google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo"
    25  	"google.golang.org/protobuf/compiler/protogen"
    26  	"google.golang.org/protobuf/internal/detrand"
    27  	"google.golang.org/protobuf/reflect/protodesc"
    28  )
    29  
    30  func init() {
    31  	// Determine repository root path.
    32  	out, err := exec.Command("git", "rev-parse", "--show-toplevel").CombinedOutput()
    33  	check(err)
    34  	repoRoot = strings.TrimSpace(string(out))
    35  
    36  	// Determine the module path.
    37  	cmd := exec.Command("go", "list", "-m", "-f", "{{.Path}}")
    38  	cmd.Dir = repoRoot
    39  	out, err = cmd.CombinedOutput()
    40  	check(err)
    41  	modulePath = strings.TrimSpace(string(out))
    42  
    43  	// When the environment variable RUN_AS_PROTOC_PLUGIN is set,
    44  	// we skip running main and instead act as a protoc plugin.
    45  	// This allows the binary to pass itself to protoc.
    46  	if plugin := os.Getenv("RUN_AS_PROTOC_PLUGIN"); plugin != "" {
    47  		// Disable deliberate output instability for generated files.
    48  		// This is reasonable since we fully control the output.
    49  		detrand.Disable()
    50  
    51  		protogen.Options{}.Run(func(gen *protogen.Plugin) error {
    52  			for _, file := range gen.Files {
    53  				if file.Generate {
    54  					gengo.GenerateVersionMarkers = false
    55  					gengo.GenerateFile(gen, file)
    56  					generateIdentifiers(gen, file)
    57  					generateSourceContextStringer(gen, file)
    58  				}
    59  			}
    60  			gen.SupportedFeatures = gengo.SupportedFeatures
    61  			return nil
    62  		})
    63  		os.Exit(0)
    64  	}
    65  }
    66  
    67  var (
    68  	run        bool
    69  	protoRoot  string
    70  	repoRoot   string
    71  	modulePath string
    72  
    73  	generatedPreamble = []string{
    74  		"// Copyright 2019 The Go Authors. All rights reserved.",
    75  		"// Use of this source code is governed by a BSD-style",
    76  		"// license that can be found in the LICENSE file.",
    77  		"",
    78  		"// Code generated by generate-protos. DO NOT EDIT.",
    79  		"",
    80  	}
    81  )
    82  
    83  func main() {
    84  	flag.BoolVar(&run, "execute", false, "Write generated files to destination.")
    85  	flag.StringVar(&protoRoot, "protoroot", os.Getenv("PROTOBUF_ROOT"), "The root of the protobuf source tree.")
    86  	flag.Parse()
    87  	if protoRoot == "" {
    88  		panic("protobuf source root is not set")
    89  	}
    90  
    91  	generateLocalProtos()
    92  	generateRemoteProtos()
    93  	generateEditionsDefaults()
    94  }
    95  
    96  func generateEditionsDefaults() {
    97  	dest := filepath.Join(repoRoot, "reflect", "protodesc", "editions_defaults.binpb")
    98  	// The enum in Go string formats to "EDITION_${EDITION}" but protoc expects
    99  	// the flag in the form "${EDITION}". To work around this, we trim the
   100  	// "EDITION_" prefix.
   101  	minEdition := strings.TrimPrefix(fmt.Sprint(protodesc.SupportedEditionsMinimum), "EDITION_")
   102  	maxEdition := strings.TrimPrefix(fmt.Sprint(protodesc.SupportedEditionsMaximum), "EDITION_")
   103  	cmd := exec.Command(
   104  		"protoc",
   105  		"--experimental_edition_defaults_out", dest,
   106  		"--experimental_edition_defaults_minimum", minEdition,
   107  		"--experimental_edition_defaults_maximum", maxEdition,
   108  		"--proto_path", protoRoot, "src/google/protobuf/descriptor.proto",
   109  	)
   110  	out, err := cmd.CombinedOutput()
   111  	if err != nil {
   112  		fmt.Printf("executing: %v\n%s\n", strings.Join(cmd.Args, " "), out)
   113  	}
   114  	check(err)
   115  }
   116  
   117  func generateLocalProtos() {
   118  	tmpDir, err := ioutil.TempDir(repoRoot, "tmp")
   119  	check(err)
   120  	defer os.RemoveAll(tmpDir)
   121  
   122  	// Generate all local proto files (except version-locked files).
   123  	dirs := []struct {
   124  		path     string
   125  		pkgPaths map[string]string // mapping of .proto path to Go package path
   126  		annotate map[string]bool   // .proto files to annotate
   127  		exclude  map[string]bool   // .proto files to exclude from generation
   128  	}{{
   129  		path:     "cmd/protoc-gen-go/testdata",
   130  		pkgPaths: map[string]string{"cmd/protoc-gen-go/testdata/nopackage/nopackage.proto": "google.golang.org/protobuf/cmd/protoc-gen-go/testdata/nopackage"},
   131  		annotate: map[string]bool{"cmd/protoc-gen-go/testdata/annotations/annotations.proto": true},
   132  	}, {
   133  		path:    "internal/testprotos",
   134  		exclude: map[string]bool{"internal/testprotos/irregular/irregular.proto": true},
   135  	}}
   136  	excludeRx := regexp.MustCompile(`legacy/.*/`)
   137  	for _, d := range dirs {
   138  		subDirs := map[string]bool{}
   139  
   140  		srcDir := filepath.Join(repoRoot, filepath.FromSlash(d.path))
   141  		filepath.Walk(srcDir, func(srcPath string, _ os.FileInfo, _ error) error {
   142  			if !strings.HasSuffix(srcPath, ".proto") || excludeRx.MatchString(srcPath) {
   143  				return nil
   144  			}
   145  			relPath, err := filepath.Rel(repoRoot, srcPath)
   146  			check(err)
   147  
   148  			srcRelPath, err := filepath.Rel(srcDir, srcPath)
   149  			check(err)
   150  			subDirs[filepath.Dir(srcRelPath)] = true
   151  
   152  			if d.exclude[filepath.ToSlash(relPath)] {
   153  				return nil
   154  			}
   155  
   156  			opts := "module=" + modulePath
   157  			for protoPath, goPkgPath := range d.pkgPaths {
   158  				opts += fmt.Sprintf(",M%v=%v", protoPath, goPkgPath)
   159  			}
   160  			if d.annotate[filepath.ToSlash(relPath)] {
   161  				opts += ",annotate_code"
   162  			}
   163  			protoc("-I"+filepath.Join(protoRoot, "src"), "-I"+repoRoot, "--go_out="+opts+":"+tmpDir, relPath)
   164  			return nil
   165  		})
   166  
   167  		// For directories in testdata, generate a test that links in all
   168  		// generated packages to ensure that it builds and initializes properly.
   169  		// This is done because "go build ./..." does not build sub-packages
   170  		// under testdata.
   171  		if filepath.Base(d.path) == "testdata" {
   172  			var imports []string
   173  			for sd := range subDirs {
   174  				imports = append(imports, fmt.Sprintf("_ %q", path.Join(modulePath, d.path, filepath.ToSlash(sd))))
   175  			}
   176  			sort.Strings(imports)
   177  
   178  			s := strings.Join(append(generatedPreamble, []string{
   179  				"package main",
   180  				"",
   181  				"import (" + strings.Join(imports, "\n") + ")",
   182  			}...), "\n")
   183  			b, err := format.Source([]byte(s))
   184  			check(err)
   185  			check(ioutil.WriteFile(filepath.Join(tmpDir, filepath.FromSlash(d.path+"/gen_test.go")), b, 0664))
   186  		}
   187  	}
   188  
   189  	syncOutput(repoRoot, tmpDir)
   190  }
   191  
   192  func generateRemoteProtos() {
   193  	tmpDir, err := ioutil.TempDir(repoRoot, "tmp")
   194  	check(err)
   195  	defer os.RemoveAll(tmpDir)
   196  
   197  	// Generate all remote proto files.
   198  	files := []struct{ prefix, path, goPkgPath string }{
   199  		// Well-known protos.
   200  		{"src", "google/protobuf/any.proto", ""},
   201  		{"src", "google/protobuf/api.proto", ""},
   202  		{"src", "google/protobuf/duration.proto", ""},
   203  		{"src", "google/protobuf/empty.proto", ""},
   204  		{"src", "google/protobuf/field_mask.proto", ""},
   205  		{"src", "google/protobuf/source_context.proto", ""},
   206  		{"src", "google/protobuf/struct.proto", ""},
   207  		{"src", "google/protobuf/timestamp.proto", ""},
   208  		{"src", "google/protobuf/type.proto", ""},
   209  		{"src", "google/protobuf/wrappers.proto", ""},
   210  
   211  		// Compiler protos.
   212  		{"src", "google/protobuf/compiler/plugin.proto", ""},
   213  		{"src", "google/protobuf/descriptor.proto", ""},
   214  
   215  		// Conformance protos.
   216  		{"", "conformance/conformance.proto", "google.golang.org/protobuf/internal/testprotos/conformance;conformance"},
   217  		{"src", "google/protobuf/test_messages_proto2.proto", "google.golang.org/protobuf/internal/testprotos/conformance;conformance"},
   218  		{"src", "google/protobuf/test_messages_proto3.proto", "google.golang.org/protobuf/internal/testprotos/conformance;conformance"},
   219  
   220  		// Benchmark protos.
   221  		// TODO: The protobuf repo no longer includes benchmarks.
   222  		//       CL removing them says they are superceded by google/fleetbench:
   223  		//         https://github.com/protocolbuffers/protobuf/commit/83c499de86224538e5d59adc3d0fa7fdb45b2c72
   224  		//       But that project's proto benchmark files are very different:
   225  		//         https://github.com/google/fleetbench/tree/main/fleetbench/proto
   226  		//       For now, commenting these out until benchmarks in this repo can be
   227  		//       reconciled with new fleetbench stuff.
   228  		//{"benchmarks", "benchmarks.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks;benchmarks"},
   229  		//{"benchmarks", "datasets/google_message1/proto2/benchmark_message1_proto2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message1/proto2;proto2"},
   230  		//{"benchmarks", "datasets/google_message1/proto3/benchmark_message1_proto3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message1/proto3;proto3"},
   231  		//{"benchmarks", "datasets/google_message2/benchmark_message2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message2;google_message2"},
   232  		//{"benchmarks", "datasets/google_message3/benchmark_message3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
   233  		//{"benchmarks", "datasets/google_message3/benchmark_message3_1.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
   234  		//{"benchmarks", "datasets/google_message3/benchmark_message3_2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
   235  		//{"benchmarks", "datasets/google_message3/benchmark_message3_3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
   236  		//{"benchmarks", "datasets/google_message3/benchmark_message3_4.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
   237  		//{"benchmarks", "datasets/google_message3/benchmark_message3_5.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
   238  		//{"benchmarks", "datasets/google_message3/benchmark_message3_6.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
   239  		//{"benchmarks", "datasets/google_message3/benchmark_message3_7.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
   240  		//{"benchmarks", "datasets/google_message3/benchmark_message3_8.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
   241  		//{"benchmarks", "datasets/google_message4/benchmark_message4.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"},
   242  		//{"benchmarks", "datasets/google_message4/benchmark_message4_1.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"},
   243  		//{"benchmarks", "datasets/google_message4/benchmark_message4_2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"},
   244  		//{"benchmarks", "datasets/google_message4/benchmark_message4_3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"},
   245  	}
   246  
   247  	opts := "module=" + modulePath
   248  	for _, file := range files {
   249  		if file.goPkgPath != "" {
   250  			opts += fmt.Sprintf(",M%v=%v", file.path, file.goPkgPath)
   251  		}
   252  	}
   253  	for _, f := range files {
   254  		protoc("-I"+filepath.Join(protoRoot, f.prefix), "--go_out="+opts+":"+tmpDir, f.path)
   255  	}
   256  
   257  	syncOutput(repoRoot, tmpDir)
   258  }
   259  
   260  func protoc(args ...string) {
   261  	// TODO: Remove --experimental_allow_proto3_optional flag.
   262  	cmd := exec.Command("protoc", "--plugin=protoc-gen-go="+os.Args[0], "--experimental_allow_proto3_optional")
   263  	cmd.Args = append(cmd.Args, args...)
   264  	cmd.Env = append(os.Environ(), "RUN_AS_PROTOC_PLUGIN=1")
   265  	out, err := cmd.CombinedOutput()
   266  	if err != nil {
   267  		fmt.Printf("executing: %v\n%s\n", strings.Join(cmd.Args, " "), out)
   268  	}
   269  	check(err)
   270  }
   271  
   272  // generateIdentifiers generates an internal package for descriptor.proto
   273  // and well-known types.
   274  func generateIdentifiers(gen *protogen.Plugin, file *protogen.File) {
   275  	if file.Desc.Package() != "google.protobuf" {
   276  		return
   277  	}
   278  
   279  	importPath := modulePath + "/internal/genid"
   280  	base := strings.TrimSuffix(path.Base(file.Desc.Path()), ".proto")
   281  	g := gen.NewGeneratedFile(importPath+"/"+base+"_gen.go", protogen.GoImportPath(importPath))
   282  	for _, s := range generatedPreamble {
   283  		g.P(s)
   284  	}
   285  	g.P("package ", path.Base(importPath))
   286  	g.P()
   287  
   288  	g.P("const ", file.GoDescriptorIdent.GoName, " = ", strconv.Quote(file.Desc.Path()))
   289  	g.P()
   290  
   291  	var processEnums func([]*protogen.Enum)
   292  	var processMessages func([]*protogen.Message)
   293  	const protoreflectPackage = protogen.GoImportPath("google.golang.org/protobuf/reflect/protoreflect")
   294  	processEnums = func(enums []*protogen.Enum) {
   295  		for _, enum := range enums {
   296  			g.P("// Full and short names for ", enum.Desc.FullName(), ".")
   297  			g.P("const (")
   298  			g.P(enum.GoIdent.GoName, "_enum_fullname = ", strconv.Quote(string(enum.Desc.FullName())))
   299  			g.P(enum.GoIdent.GoName, "_enum_name = ", strconv.Quote(string(enum.Desc.Name())))
   300  			g.P(")")
   301  			g.P()
   302  		}
   303  	}
   304  	processMessages = func(messages []*protogen.Message) {
   305  		for _, message := range messages {
   306  			g.P("// Names for ", message.Desc.FullName(), ".")
   307  			g.P("const (")
   308  			g.P(message.GoIdent.GoName, "_message_name ", protoreflectPackage.Ident("Name"), " = ", strconv.Quote(string(message.Desc.Name())))
   309  			g.P(message.GoIdent.GoName, "_message_fullname ", protoreflectPackage.Ident("FullName"), " = ", strconv.Quote(string(message.Desc.FullName())))
   310  			g.P(")")
   311  			g.P()
   312  
   313  			if len(message.Fields) > 0 {
   314  				g.P("// Field names for ", message.Desc.FullName(), ".")
   315  				g.P("const (")
   316  				for _, field := range message.Fields {
   317  					g.P(message.GoIdent.GoName, "_", field.GoName, "_field_name ", protoreflectPackage.Ident("Name"), " = ", strconv.Quote(string(field.Desc.Name())))
   318  				}
   319  				g.P()
   320  				for _, field := range message.Fields {
   321  					g.P(message.GoIdent.GoName, "_", field.GoName, "_field_fullname ", protoreflectPackage.Ident("FullName"), " = ", strconv.Quote(string(field.Desc.FullName())))
   322  				}
   323  				g.P(")")
   324  				g.P()
   325  
   326  				g.P("// Field numbers for ", message.Desc.FullName(), ".")
   327  				g.P("const (")
   328  				for _, field := range message.Fields {
   329  					g.P(message.GoIdent.GoName, "_", field.GoName, "_field_number ", protoreflectPackage.Ident("FieldNumber"), " = ", field.Desc.Number())
   330  				}
   331  				g.P(")")
   332  				g.P()
   333  			}
   334  
   335  			if len(message.Oneofs) > 0 {
   336  				g.P("// Oneof names for ", message.Desc.FullName(), ".")
   337  				g.P("const (")
   338  				for _, oneof := range message.Oneofs {
   339  					g.P(message.GoIdent.GoName, "_", oneof.GoName, "_oneof_name ", protoreflectPackage.Ident("Name"), " = ", strconv.Quote(string(oneof.Desc.Name())))
   340  				}
   341  				g.P()
   342  				for _, oneof := range message.Oneofs {
   343  					g.P(message.GoIdent.GoName, "_", oneof.GoName, "_oneof_fullname ", protoreflectPackage.Ident("FullName"), " = ", strconv.Quote(string(oneof.Desc.FullName())))
   344  				}
   345  				g.P(")")
   346  				g.P()
   347  			}
   348  
   349  			processEnums(message.Enums)
   350  			processMessages(message.Messages)
   351  		}
   352  	}
   353  	processEnums(file.Enums)
   354  	processMessages(file.Messages)
   355  }
   356  
   357  // generateSourceContextStringer generates the implementation for the
   358  // protoreflect.SourcePath.String method by using information present
   359  // in the descriptor.proto.
   360  func generateSourceContextStringer(gen *protogen.Plugin, file *protogen.File) {
   361  	if file.Desc.Path() != "google/protobuf/descriptor.proto" {
   362  		return
   363  	}
   364  
   365  	importPath := modulePath + "/reflect/protoreflect"
   366  	g := gen.NewGeneratedFile(importPath+"/source_gen.go", protogen.GoImportPath(importPath))
   367  	for _, s := range generatedPreamble {
   368  		g.P(s)
   369  	}
   370  	g.P("package ", path.Base(importPath))
   371  	g.P()
   372  
   373  	var messages []*protogen.Message
   374  	for _, message := range file.Messages {
   375  		if message.Desc.Name() == "FileDescriptorProto" {
   376  			messages = append(messages, message)
   377  		}
   378  	}
   379  	seen := make(map[*protogen.Message]bool)
   380  
   381  	for len(messages) > 0 {
   382  		m := messages[0]
   383  		messages = messages[1:]
   384  		if seen[m] {
   385  			continue
   386  		}
   387  		seen[m] = true
   388  
   389  		g.P("func (p *SourcePath) append", m.GoIdent.GoName, "(b []byte) []byte {")
   390  		g.P("if len(*p) == 0 { return b }")
   391  		g.P("switch (*p)[0] {")
   392  		for _, f := range m.Fields {
   393  			g.P("case ", f.Desc.Number(), ":")
   394  			var cardinality string
   395  			switch {
   396  			case f.Desc.IsMap():
   397  				panic("maps are not supported")
   398  			case f.Desc.IsList():
   399  				cardinality = "Repeated"
   400  			default:
   401  				cardinality = "Singular"
   402  			}
   403  			nextAppender := "nil"
   404  			if f.Message != nil {
   405  				nextAppender = "(*SourcePath).append" + f.Message.GoIdent.GoName
   406  				messages = append(messages, f.Message)
   407  			}
   408  			g.P("b = p.append", cardinality, "Field(b, ", strconv.Quote(string(f.Desc.Name())), ", ", nextAppender, ")")
   409  		}
   410  		g.P("}")
   411  		g.P("return b")
   412  		g.P("}")
   413  		g.P()
   414  	}
   415  }
   416  
   417  func syncOutput(dstDir, srcDir string) {
   418  	filepath.Walk(srcDir, func(srcPath string, _ os.FileInfo, _ error) error {
   419  		if !strings.HasSuffix(srcPath, ".go") && !strings.HasSuffix(srcPath, ".meta") {
   420  			return nil
   421  		}
   422  		relPath, err := filepath.Rel(srcDir, srcPath)
   423  		check(err)
   424  		dstPath := filepath.Join(dstDir, relPath)
   425  
   426  		if run {
   427  			if copyFile(dstPath, srcPath) {
   428  				fmt.Println("#", relPath)
   429  			}
   430  		} else {
   431  			cmd := exec.Command("diff", dstPath, srcPath, "-N", "-u")
   432  			cmd.Stdout = os.Stdout
   433  			cmd.Run()
   434  		}
   435  		return nil
   436  	})
   437  }
   438  
   439  func copyFile(dstPath, srcPath string) (changed bool) {
   440  	src, err := ioutil.ReadFile(srcPath)
   441  	check(err)
   442  	check(os.MkdirAll(filepath.Dir(dstPath), 0775))
   443  	dst, _ := ioutil.ReadFile(dstPath)
   444  	if bytes.Equal(src, dst) {
   445  		return false
   446  	}
   447  	check(ioutil.WriteFile(dstPath, src, 0664))
   448  	return true
   449  }
   450  
   451  func check(err error) {
   452  	if err != nil {
   453  		panic(err)
   454  	}
   455  }
   456
View as plain text