buildtool.go

Documentation: cmd/dist

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Build toolchain using Go bootstrap version.
     6  //
     7  // The general strategy is to copy the source files we need into
     8  // a new GOPATH workspace, adjust import paths appropriately,
     9  // invoke the Go bootstrap toolchains go command to build those sources,
    10  // and then copy the binaries back.
    11  
    12  package main
    13  
    14  import (
    15  	"fmt"
    16  	"os"
    17  	"path/filepath"
    18  	"regexp"
    19  	"strings"
    20  )
    21  
    22  // bootstrapDirs is a list of directories holding code that must be
    23  // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
    24  // All directories in this list are relative to and must be below $GOROOT/src.
    25  //
    26  // The list has two kinds of entries: names beginning with cmd/ with
    27  // no other slashes, which are commands, and other paths, which are packages
    28  // supporting the commands. Packages in the standard library can be listed
    29  // if a newer copy needs to be substituted for the Go bootstrap copy when used
    30  // by the command packages. Paths ending with /... automatically
    31  // include all packages within subdirectories as well.
    32  // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
    33  var bootstrapDirs = []string{
    34  	"cmp",
    35  	"cmd/asm",
    36  	"cmd/asm/internal/...",
    37  	"cmd/cgo",
    38  	"cmd/compile",
    39  	"cmd/compile/internal/...",
    40  	"cmd/internal/archive",
    41  	"cmd/internal/bio",
    42  	"cmd/internal/codesign",
    43  	"cmd/internal/dwarf",
    44  	"cmd/internal/edit",
    45  	"cmd/internal/gcprog",
    46  	"cmd/internal/goobj",
    47  	"cmd/internal/notsha256",
    48  	"cmd/internal/obj/...",
    49  	"cmd/internal/objabi",
    50  	"cmd/internal/pkgpath",
    51  	"cmd/internal/quoted",
    52  	"cmd/internal/src",
    53  	"cmd/internal/sys",
    54  	"cmd/link",
    55  	"cmd/link/internal/...",
    56  	"compress/flate",
    57  	"compress/zlib",
    58  	"container/heap",
    59  	"debug/dwarf",
    60  	"debug/elf",
    61  	"debug/macho",
    62  	"debug/pe",
    63  	"go/build/constraint",
    64  	"go/constant",
    65  	"go/version",
    66  	"internal/abi",
    67  	"internal/coverage",
    68  	"cmd/internal/cov/covcmd",
    69  	"internal/bisect",
    70  	"internal/buildcfg",
    71  	"internal/goarch",
    72  	"internal/godebugs",
    73  	"internal/goexperiment",
    74  	"internal/goroot",
    75  	"internal/gover",
    76  	"internal/goversion",
    77  	// internal/lazyregexp is provided by Go 1.17, which permits it to
    78  	// be imported by other packages in this list, but is not provided
    79  	// by the Go 1.17 version of gccgo. It's on this list only to
    80  	// support gccgo, and can be removed if we require gccgo 14 or later.
    81  	"internal/lazyregexp",
    82  	"internal/pkgbits",
    83  	"internal/platform",
    84  	"internal/profile",
    85  	"internal/race",
    86  	"internal/saferio",
    87  	"internal/syscall/unix",
    88  	"internal/types/errors",
    89  	"internal/unsafeheader",
    90  	"internal/xcoff",
    91  	"internal/zstd",
    92  	"math/bits",
    93  	"sort",
    94  }
    95  
    96  // File prefixes that are ignored by go/build anyway, and cause
    97  // problems with editor generated temporary files (#18931).
    98  var ignorePrefixes = []string{
    99  	".",
   100  	"_",
   101  	"#",
   102  }
   103  
   104  // File suffixes that use build tags introduced since Go 1.17.
   105  // These must not be copied into the bootstrap build directory.
   106  // Also ignore test files.
   107  var ignoreSuffixes = []string{
   108  	"_test.s",
   109  	"_test.go",
   110  	// Skip PGO profile. No need to build toolchain1 compiler
   111  	// with PGO. And as it is not a text file the import path
   112  	// rewrite will break it.
   113  	".pgo",
   114  }
   115  
   116  var tryDirs = []string{
   117  	"sdk/go1.17",
   118  	"go1.17",
   119  }
   120  
   121  func bootstrapBuildTools() {
   122  	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
   123  	if goroot_bootstrap == "" {
   124  		home := os.Getenv("HOME")
   125  		goroot_bootstrap = pathf("%s/go1.4", home)
   126  		for _, d := range tryDirs {
   127  			if p := pathf("%s/%s", home, d); isdir(p) {
   128  				goroot_bootstrap = p
   129  			}
   130  		}
   131  	}
   132  	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
   133  
   134  	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
   135  	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
   136  
   137  	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
   138  	// We use a subdirectory of $GOROOT/pkg because that's the
   139  	// space within $GOROOT where we store all generated objects.
   140  	// We could use a temporary directory outside $GOROOT instead,
   141  	// but it is easier to debug on failure if the files are in a known location.
   142  	workspace := pathf("%s/pkg/bootstrap", goroot)
   143  	xremoveall(workspace)
   144  	xatexit(func() { xremoveall(workspace) })
   145  	base := pathf("%s/src/bootstrap", workspace)
   146  	xmkdirall(base)
   147  
   148  	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
   149  	writefile("module bootstrap\ngo 1.20\n", pathf("%s/%s", base, "go.mod"), 0)
   150  	for _, dir := range bootstrapDirs {
   151  		recurse := strings.HasSuffix(dir, "/...")
   152  		dir = strings.TrimSuffix(dir, "/...")
   153  		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   154  			if err != nil {
   155  				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
   156  			}
   157  
   158  			name := filepath.Base(path)
   159  			src := pathf("%s/src/%s", goroot, path)
   160  			dst := pathf("%s/%s", base, path)
   161  
   162  			if info.IsDir() {
   163  				if !recurse && path != dir || name == "testdata" {
   164  					return filepath.SkipDir
   165  				}
   166  
   167  				xmkdirall(dst)
   168  				if path == "cmd/cgo" {
   169  					// Write to src because we need the file both for bootstrap
   170  					// and for later in the main build.
   171  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
   172  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
   173  				}
   174  				return nil
   175  			}
   176  
   177  			for _, pre := range ignorePrefixes {
   178  				if strings.HasPrefix(name, pre) {
   179  					return nil
   180  				}
   181  			}
   182  			for _, suf := range ignoreSuffixes {
   183  				if strings.HasSuffix(name, suf) {
   184  					return nil
   185  				}
   186  			}
   187  
   188  			text := bootstrapRewriteFile(src)
   189  			writefile(text, dst, 0)
   190  			return nil
   191  		})
   192  	}
   193  
   194  	// Set up environment for invoking Go bootstrap toolchains go command.
   195  	// GOROOT points at Go bootstrap GOROOT,
   196  	// GOPATH points at our bootstrap workspace,
   197  	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
   198  	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
   199  	// so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
   200  	// Restore GOROOT, GOPATH, and GOBIN when done.
   201  	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
   202  	// because setup will take care of those when bootstrapBuildTools returns.
   203  
   204  	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
   205  	os.Setenv("GOROOT", goroot_bootstrap)
   206  
   207  	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
   208  	os.Setenv("GOPATH", workspace)
   209  
   210  	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
   211  	os.Setenv("GOBIN", "")
   212  
   213  	os.Setenv("GOOS", "")
   214  	os.Setenv("GOHOSTOS", "")
   215  	os.Setenv("GOARCH", "")
   216  	os.Setenv("GOHOSTARCH", "")
   217  
   218  	// Run Go bootstrap to build binaries.
   219  	// Use the math_big_pure_go build tag to disable the assembly in math/big
   220  	// which may contain unsupported instructions.
   221  	// Use the purego build tag to disable other assembly code,
   222  	// such as in cmd/internal/notsha256.
   223  	cmd := []string{
   224  		pathf("%s/bin/go", goroot_bootstrap),
   225  		"install",
   226  		"-tags=math_big_pure_go compiler_bootstrap purego",
   227  	}
   228  	if vflag > 0 {
   229  		cmd = append(cmd, "-v")
   230  	}
   231  	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
   232  		cmd = append(cmd, "-toolexec="+tool)
   233  	}
   234  	cmd = append(cmd, "bootstrap/cmd/...")
   235  	run(base, ShowOutput|CheckExit, cmd...)
   236  
   237  	// Copy binaries into tool binary directory.
   238  	for _, name := range bootstrapDirs {
   239  		if !strings.HasPrefix(name, "cmd/") {
   240  			continue
   241  		}
   242  		name = name[len("cmd/"):]
   243  		if !strings.Contains(name, "/") {
   244  			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
   245  		}
   246  	}
   247  
   248  	if vflag > 0 {
   249  		xprintf("\n")
   250  	}
   251  }
   252  
   253  var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
   254  
   255  // isUnneededSSARewriteFile reports whether srcFile is a
   256  // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
   257  // architecture that isn't for the given GOARCH.
   258  //
   259  // When unneeded is true archCaps is the rewrite base filename without
   260  // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
   261  func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
   262  	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
   263  		return "", false
   264  	}
   265  	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
   266  	if fileArch == "" {
   267  		return "", false
   268  	}
   269  	b := fileArch[0]
   270  	if b == '_' || ('a' <= b && b <= 'z') {
   271  		return "", false
   272  	}
   273  	archCaps = fileArch
   274  	fileArch = strings.ToLower(fileArch)
   275  	fileArch = strings.TrimSuffix(fileArch, "splitload")
   276  	fileArch = strings.TrimSuffix(fileArch, "latelower")
   277  	if fileArch == goArch {
   278  		return "", false
   279  	}
   280  	if fileArch == strings.TrimSuffix(goArch, "le") {
   281  		return "", false
   282  	}
   283  	return archCaps, true
   284  }
   285  
   286  func bootstrapRewriteFile(srcFile string) string {
   287  	// During bootstrap, generate dummy rewrite files for
   288  	// irrelevant architectures. We only need to build a bootstrap
   289  	// binary that works for the current gohostarch.
   290  	// This saves 6+ seconds of bootstrap.
   291  	if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
   292  		return fmt.Sprintf(`%spackage ssa
   293  
   294  func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
   295  func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
   296  `, generatedHeader, archCaps, archCaps)
   297  	}
   298  
   299  	return bootstrapFixImports(srcFile)
   300  }
   301  
   302  func bootstrapFixImports(srcFile string) string {
   303  	text := readfile(srcFile)
   304  	if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
   305  		text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
   306  	}
   307  	lines := strings.SplitAfter(text, "\n")
   308  	inBlock := false
   309  	for i, line := range lines {
   310  		if strings.HasPrefix(line, "import (") {
   311  			inBlock = true
   312  			continue
   313  		}
   314  		if inBlock && strings.HasPrefix(line, ")") {
   315  			inBlock = false
   316  			continue
   317  		}
   318  		if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
   319  			inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"") || strings.HasPrefix(line, "\trtabi \"")) {
   320  			line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
   321  			for _, dir := range bootstrapDirs {
   322  				if strings.HasPrefix(dir, "cmd/") {
   323  					continue
   324  				}
   325  				line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
   326  			}
   327  			lines[i] = line
   328  		}
   329  	}
   330  
   331  	lines[0] = generatedHeader + "// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
   332  
   333  	return strings.Join(lines, "")
   334  }
   335
View as plain text