Source file src/cmd/go/internal/modfetch/fetch.go

Documentation: cmd/go/internal/modfetch

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"archive/zip"
     9  	"bytes"
    10  	"context"
    11  	"crypto/sha256"
    12  	"encoding/base64"
    13  	"errors"
    14  	"fmt"
    15  	"io"
    16  	"io/fs"
    17  	"os"
    18  	"path/filepath"
    19  	"sort"
    20  	"strings"
    21  	"sync"
    22  
    23  	"cmd/go/internal/base"
    24  	"cmd/go/internal/cfg"
    25  	"cmd/go/internal/fsys"
    26  	"cmd/go/internal/gover"
    27  	"cmd/go/internal/lockedfile"
    28  	"cmd/go/internal/par"
    29  	"cmd/go/internal/robustio"
    30  	"cmd/go/internal/str"
    31  	"cmd/go/internal/trace"
    32  
    33  	"golang.org/x/mod/module"
    34  	"golang.org/x/mod/sumdb/dirhash"
    35  	modzip "golang.org/x/mod/zip"
    36  )
    37  
    38  var downloadCache par.ErrCache[module.Version, string] // version → directory
    39  
    40  var ErrToolchain = errors.New("internal error: invalid operation on toolchain module")
    41  
    42  // Download downloads the specific module version to the
    43  // local download cache and returns the name of the directory
    44  // corresponding to the root of the module's file tree.
    45  func Download(ctx context.Context, mod module.Version) (dir string, err error) {
    46  	if gover.IsToolchain(mod.Path) {
    47  		return "", ErrToolchain
    48  	}
    49  	if err := checkCacheDir(ctx); err != nil {
    50  		base.Fatal(err)
    51  	}
    52  
    53  	// The par.Cache here avoids duplicate work.
    54  	return downloadCache.Do(mod, func() (string, error) {
    55  		dir, err := download(ctx, mod)
    56  		if err != nil {
    57  			return "", err
    58  		}
    59  		checkMod(ctx, mod)
    60  
    61  		// If go.mod exists (not an old legacy module), check version is not too new.
    62  		if data, err := os.ReadFile(filepath.Join(dir, "go.mod")); err == nil {
    63  			goVersion := gover.GoModLookup(data, "go")
    64  			if gover.Compare(goVersion, gover.Local()) > 0 {
    65  				return "", &gover.TooNewError{What: mod.String(), GoVersion: goVersion}
    66  			}
    67  		} else if !errors.Is(err, fs.ErrNotExist) {
    68  			return "", err
    69  		}
    70  
    71  		return dir, nil
    72  	})
    73  }
    74  
    75  func download(ctx context.Context, mod module.Version) (dir string, err error) {
    76  	ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String())
    77  	defer span.Done()
    78  
    79  	dir, err = DownloadDir(ctx, mod)
    80  	if err == nil {
    81  		// The directory has already been completely extracted (no .partial file exists).
    82  		return dir, nil
    83  	} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
    84  		return "", err
    85  	}
    86  
    87  	// To avoid cluttering the cache with extraneous files,
    88  	// DownloadZip uses the same lockfile as Download.
    89  	// Invoke DownloadZip before locking the file.
    90  	zipfile, err := DownloadZip(ctx, mod)
    91  	if err != nil {
    92  		return "", err
    93  	}
    94  
    95  	unlock, err := lockVersion(ctx, mod)
    96  	if err != nil {
    97  		return "", err
    98  	}
    99  	defer unlock()
   100  
   101  	ctx, span = trace.StartSpan(ctx, "unzip "+zipfile)
   102  	defer span.Done()
   103  
   104  	// Check whether the directory was populated while we were waiting on the lock.
   105  	_, dirErr := DownloadDir(ctx, mod)
   106  	if dirErr == nil {
   107  		return dir, nil
   108  	}
   109  	_, dirExists := dirErr.(*DownloadDirPartialError)
   110  
   111  	// Clean up any remaining temporary directories created by old versions
   112  	// (before 1.16), as well as partially extracted directories (indicated by
   113  	// DownloadDirPartialError, usually because of a .partial file). This is only
   114  	// safe to do because the lock file ensures that their writers are no longer
   115  	// active.
   116  	parentDir := filepath.Dir(dir)
   117  	tmpPrefix := filepath.Base(dir) + ".tmp-"
   118  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(parentDir), str.QuoteGlob(tmpPrefix)+"*")); err == nil {
   119  		for _, path := range old {
   120  			RemoveAll(path) // best effort
   121  		}
   122  	}
   123  	if dirExists {
   124  		if err := RemoveAll(dir); err != nil {
   125  			return "", err
   126  		}
   127  	}
   128  
   129  	partialPath, err := CachePath(ctx, mod, "partial")
   130  	if err != nil {
   131  		return "", err
   132  	}
   133  
   134  	// Extract the module zip directory at its final location.
   135  	//
   136  	// To prevent other processes from reading the directory if we crash,
   137  	// create a .partial file before extracting the directory, and delete
   138  	// the .partial file afterward (all while holding the lock).
   139  	//
   140  	// Before Go 1.16, we extracted to a temporary directory with a random name
   141  	// then renamed it into place with os.Rename. On Windows, this failed with
   142  	// ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner)
   143  	// opened files in the temporary directory.
   144  	//
   145  	// Go 1.14.2 and higher respect .partial files. Older versions may use
   146  	// partially extracted directories. 'go mod verify' can detect this,
   147  	// and 'go clean -modcache' can fix it.
   148  	if err := os.MkdirAll(parentDir, 0777); err != nil {
   149  		return "", err
   150  	}
   151  	if err := os.WriteFile(partialPath, nil, 0666); err != nil {
   152  		return "", err
   153  	}
   154  	if err := modzip.Unzip(dir, mod, zipfile); err != nil {
   155  		fmt.Fprintf(os.Stderr, "-> %s\n", err)
   156  		if rmErr := RemoveAll(dir); rmErr == nil {
   157  			os.Remove(partialPath)
   158  		}
   159  		return "", err
   160  	}
   161  	if err := os.Remove(partialPath); err != nil {
   162  		return "", err
   163  	}
   164  
   165  	if !cfg.ModCacheRW {
   166  		makeDirsReadOnly(dir)
   167  	}
   168  	return dir, nil
   169  }
   170  
   171  var downloadZipCache par.ErrCache[module.Version, string]
   172  
   173  // DownloadZip downloads the specific module version to the
   174  // local zip cache and returns the name of the zip file.
   175  func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) {
   176  	// The par.Cache here avoids duplicate work.
   177  	return downloadZipCache.Do(mod, func() (string, error) {
   178  		zipfile, err := CachePath(ctx, mod, "zip")
   179  		if err != nil {
   180  			return "", err
   181  		}
   182  		ziphashfile := zipfile + "hash"
   183  
   184  		// Return without locking if the zip and ziphash files exist.
   185  		if _, err := os.Stat(zipfile); err == nil {
   186  			if _, err := os.Stat(ziphashfile); err == nil {
   187  				return zipfile, nil
   188  			}
   189  		}
   190  
   191  		// The zip or ziphash file does not exist. Acquire the lock and create them.
   192  		if cfg.CmdName != "mod download" {
   193  			vers := mod.Version
   194  			if mod.Path == "golang.org/toolchain" {
   195  				// Shorten v0.0.1-go1.13.1.darwin-amd64 to go1.13.1.darwin-amd64
   196  				_, vers, _ = strings.Cut(vers, "-")
   197  				if i := strings.LastIndex(vers, "."); i >= 0 {
   198  					goos, goarch, _ := strings.Cut(vers[i+1:], "-")
   199  					vers = vers[:i] + " (" + goos + "/" + goarch + ")"
   200  				}
   201  				fmt.Fprintf(os.Stderr, "go: downloading %s\n", vers)
   202  			} else {
   203  				fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, vers)
   204  			}
   205  		}
   206  		unlock, err := lockVersion(ctx, mod)
   207  		if err != nil {
   208  			return "", err
   209  		}
   210  		defer unlock()
   211  
   212  		if err := downloadZip(ctx, mod, zipfile); err != nil {
   213  			return "", err
   214  		}
   215  		return zipfile, nil
   216  	})
   217  }
   218  
   219  func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) {
   220  	ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile)
   221  	defer span.Done()
   222  
   223  	// Double-check that the zipfile was not created while we were waiting for
   224  	// the lock in DownloadZip.
   225  	ziphashfile := zipfile + "hash"
   226  	var zipExists, ziphashExists bool
   227  	if _, err := os.Stat(zipfile); err == nil {
   228  		zipExists = true
   229  	}
   230  	if _, err := os.Stat(ziphashfile); err == nil {
   231  		ziphashExists = true
   232  	}
   233  	if zipExists && ziphashExists {
   234  		return nil
   235  	}
   236  
   237  	// Create parent directories.
   238  	if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
   239  		return err
   240  	}
   241  
   242  	// Clean up any remaining tempfiles from previous runs.
   243  	// This is only safe to do because the lock file ensures that their
   244  	// writers are no longer active.
   245  	tmpPattern := filepath.Base(zipfile) + "*.tmp"
   246  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(filepath.Dir(zipfile)), tmpPattern)); err == nil {
   247  		for _, path := range old {
   248  			os.Remove(path) // best effort
   249  		}
   250  	}
   251  
   252  	// If the zip file exists, the ziphash file must have been deleted
   253  	// or lost after a file system crash. Re-hash the zip without downloading.
   254  	if zipExists {
   255  		return hashZip(mod, zipfile, ziphashfile)
   256  	}
   257  
   258  	// From here to the os.Rename call below is functionally almost equivalent to
   259  	// renameio.WriteToFile, with one key difference: we want to validate the
   260  	// contents of the file (by hashing it) before we commit it. Because the file
   261  	// is zip-compressed, we need an actual file — or at least an io.ReaderAt — to
   262  	// validate it: we can't just tee the stream as we write it.
   263  	f, err := tempFile(ctx, filepath.Dir(zipfile), filepath.Base(zipfile), 0666)
   264  	if err != nil {
   265  		return err
   266  	}
   267  	defer func() {
   268  		if err != nil {
   269  			f.Close()
   270  			os.Remove(f.Name())
   271  		}
   272  	}()
   273  
   274  	var unrecoverableErr error
   275  	err = TryProxies(func(proxy string) error {
   276  		if unrecoverableErr != nil {
   277  			return unrecoverableErr
   278  		}
   279  		repo := Lookup(ctx, proxy, mod.Path)
   280  		err := repo.Zip(ctx, f, mod.Version)
   281  		if err != nil {
   282  			// Zip may have partially written to f before failing.
   283  			// (Perhaps the server crashed while sending the file?)
   284  			// Since we allow fallback on error in some cases, we need to fix up the
   285  			// file to be empty again for the next attempt.
   286  			if _, err := f.Seek(0, io.SeekStart); err != nil {
   287  				unrecoverableErr = err
   288  				return err
   289  			}
   290  			if err := f.Truncate(0); err != nil {
   291  				unrecoverableErr = err
   292  				return err
   293  			}
   294  		}
   295  		return err
   296  	})
   297  	if err != nil {
   298  		return err
   299  	}
   300  
   301  	// Double-check that the paths within the zip file are well-formed.
   302  	//
   303  	// TODO(bcmills): There is a similar check within the Unzip function. Can we eliminate one?
   304  	fi, err := f.Stat()
   305  	if err != nil {
   306  		return err
   307  	}
   308  	z, err := zip.NewReader(f, fi.Size())
   309  	if err != nil {
   310  		return err
   311  	}
   312  	prefix := mod.Path + "@" + mod.Version + "/"
   313  	for _, f := range z.File {
   314  		if !strings.HasPrefix(f.Name, prefix) {
   315  			return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], f.Name)
   316  		}
   317  	}
   318  
   319  	if err := f.Close(); err != nil {
   320  		return err
   321  	}
   322  
   323  	// Hash the zip file and check the sum before renaming to the final location.
   324  	if err := hashZip(mod, f.Name(), ziphashfile); err != nil {
   325  		return err
   326  	}
   327  	if err := os.Rename(f.Name(), zipfile); err != nil {
   328  		return err
   329  	}
   330  
   331  	// TODO(bcmills): Should we make the .zip and .ziphash files read-only to discourage tampering?
   332  
   333  	return nil
   334  }
   335  
   336  // hashZip reads the zip file opened in f, then writes the hash to ziphashfile,
   337  // overwriting that file if it exists.
   338  //
   339  // If the hash does not match go.sum (or the sumdb if enabled), hashZip returns
   340  // an error and does not write ziphashfile.
   341  func hashZip(mod module.Version, zipfile, ziphashfile string) (err error) {
   342  	hash, err := dirhash.HashZip(zipfile, dirhash.DefaultHash)
   343  	if err != nil {
   344  		return err
   345  	}
   346  	if err := checkModSum(mod, hash); err != nil {
   347  		return err
   348  	}
   349  	hf, err := lockedfile.Create(ziphashfile)
   350  	if err != nil {
   351  		return err
   352  	}
   353  	defer func() {
   354  		if closeErr := hf.Close(); err == nil && closeErr != nil {
   355  			err = closeErr
   356  		}
   357  	}()
   358  	if err := hf.Truncate(int64(len(hash))); err != nil {
   359  		return err
   360  	}
   361  	if _, err := hf.WriteAt([]byte(hash), 0); err != nil {
   362  		return err
   363  	}
   364  	return nil
   365  }
   366  
   367  // makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
   368  // and its transitive contents.
   369  func makeDirsReadOnly(dir string) {
   370  	type pathMode struct {
   371  		path string
   372  		mode fs.FileMode
   373  	}
   374  	var dirs []pathMode // in lexical order
   375  	filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
   376  		if err == nil && d.IsDir() {
   377  			info, err := d.Info()
   378  			if err == nil && info.Mode()&0222 != 0 {
   379  				dirs = append(dirs, pathMode{path, info.Mode()})
   380  			}
   381  		}
   382  		return nil
   383  	})
   384  
   385  	// Run over list backward to chmod children before parents.
   386  	for i := len(dirs) - 1; i >= 0; i-- {
   387  		os.Chmod(dirs[i].path, dirs[i].mode&^0222)
   388  	}
   389  }
   390  
   391  // RemoveAll removes a directory written by Download or Unzip, first applying
   392  // any permission changes needed to do so.
   393  func RemoveAll(dir string) error {
   394  	// Module cache has 0555 directories; make them writable in order to remove content.
   395  	filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error {
   396  		if err != nil {
   397  			return nil // ignore errors walking in file system
   398  		}
   399  		if info.IsDir() {
   400  			os.Chmod(path, 0777)
   401  		}
   402  		return nil
   403  	})
   404  	return robustio.RemoveAll(dir)
   405  }
   406  
   407  var GoSumFile string             // path to go.sum; set by package modload
   408  var WorkspaceGoSumFiles []string // path to module go.sums in workspace; set by package modload
   409  
   410  type modSum struct {
   411  	mod module.Version
   412  	sum string
   413  }
   414  
   415  var goSum struct {
   416  	mu        sync.Mutex
   417  	m         map[module.Version][]string            // content of go.sum file
   418  	w         map[string]map[module.Version][]string // sum file in workspace -> content of that sum file
   419  	status    map[modSum]modSumStatus                // state of sums in m
   420  	overwrite bool                                   // if true, overwrite go.sum without incorporating its contents
   421  	enabled   bool                                   // whether to use go.sum at all
   422  }
   423  
   424  type modSumStatus struct {
   425  	used, dirty bool
   426  }
   427  
   428  // Reset resets globals in the modfetch package, so previous loads don't affect
   429  // contents of go.sum files.
   430  func Reset() {
   431  	GoSumFile = ""
   432  	WorkspaceGoSumFiles = nil
   433  
   434  	// Uses of lookupCache and downloadCache both can call checkModSum,
   435  	// which in turn sets the used bit on goSum.status for modules.
   436  	// Reset them so used can be computed properly.
   437  	lookupCache = par.Cache[lookupCacheKey, Repo]{}
   438  	downloadCache = par.ErrCache[module.Version, string]{}
   439  
   440  	// Clear all fields on goSum. It will be initialized later
   441  	goSum.mu.Lock()
   442  	goSum.m = nil
   443  	goSum.w = nil
   444  	goSum.status = nil
   445  	goSum.overwrite = false
   446  	goSum.enabled = false
   447  	goSum.mu.Unlock()
   448  }
   449  
   450  // initGoSum initializes the go.sum data.
   451  // The boolean it returns reports whether the
   452  // use of go.sum is now enabled.
   453  // The goSum lock must be held.
   454  func initGoSum() (bool, error) {
   455  	if GoSumFile == "" {
   456  		return false, nil
   457  	}
   458  	if goSum.m != nil {
   459  		return true, nil
   460  	}
   461  
   462  	goSum.m = make(map[module.Version][]string)
   463  	goSum.status = make(map[modSum]modSumStatus)
   464  	goSum.w = make(map[string]map[module.Version][]string)
   465  
   466  	for _, f := range WorkspaceGoSumFiles {
   467  		goSum.w[f] = make(map[module.Version][]string)
   468  		_, err := readGoSumFile(goSum.w[f], f)
   469  		if err != nil {
   470  			return false, err
   471  		}
   472  	}
   473  
   474  	enabled, err := readGoSumFile(goSum.m, GoSumFile)
   475  	goSum.enabled = enabled
   476  	return enabled, err
   477  }
   478  
   479  func readGoSumFile(dst map[module.Version][]string, file string) (bool, error) {
   480  	var (
   481  		data []byte
   482  		err  error
   483  	)
   484  	if actualSumFile, ok := fsys.OverlayPath(file); ok {
   485  		// Don't lock go.sum if it's part of the overlay.
   486  		// On Plan 9, locking requires chmod, and we don't want to modify any file
   487  		// in the overlay. See #44700.
   488  		data, err = os.ReadFile(actualSumFile)
   489  	} else {
   490  		data, err = lockedfile.Read(file)
   491  	}
   492  	if err != nil && !os.IsNotExist(err) {
   493  		return false, err
   494  	}
   495  	readGoSum(dst, file, data)
   496  
   497  	return true, nil
   498  }
   499  
   500  // emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
   501  // A bug caused us to write these into go.sum files for non-modules.
   502  // We detect and remove them.
   503  const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="
   504  
   505  // readGoSum parses data, which is the content of file,
   506  // and adds it to goSum.m. The goSum lock must be held.
   507  func readGoSum(dst map[module.Version][]string, file string, data []byte) {
   508  	lineno := 0
   509  	for len(data) > 0 {
   510  		var line []byte
   511  		lineno++
   512  		i := bytes.IndexByte(data, '\n')
   513  		if i < 0 {
   514  			line, data = data, nil
   515  		} else {
   516  			line, data = data[:i], data[i+1:]
   517  		}
   518  		f := strings.Fields(string(line))
   519  		if len(f) == 0 {
   520  			// blank line; skip it
   521  			continue
   522  		}
   523  		if len(f) != 3 {
   524  			if cfg.CmdName == "mod tidy" {
   525  				// ignore malformed line so that go mod tidy can fix go.sum
   526  				continue
   527  			} else {
   528  				base.Fatalf("malformed go.sum:\n%s:%d: wrong number of fields %v\n", file, lineno, len(f))
   529  			}
   530  		}
   531  		if f[2] == emptyGoModHash {
   532  			// Old bug; drop it.
   533  			continue
   534  		}
   535  		mod := module.Version{Path: f[0], Version: f[1]}
   536  		dst[mod] = append(dst[mod], f[2])
   537  	}
   538  }
   539  
   540  // HaveSum returns true if the go.sum file contains an entry for mod.
   541  // The entry's hash must be generated with a known hash algorithm.
   542  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   543  // .mod and .zip files.
   544  func HaveSum(mod module.Version) bool {
   545  	goSum.mu.Lock()
   546  	defer goSum.mu.Unlock()
   547  	inited, err := initGoSum()
   548  	if err != nil || !inited {
   549  		return false
   550  	}
   551  	for _, goSums := range goSum.w {
   552  		for _, h := range goSums[mod] {
   553  			if !strings.HasPrefix(h, "h1:") {
   554  				continue
   555  			}
   556  			if !goSum.status[modSum{mod, h}].dirty {
   557  				return true
   558  			}
   559  		}
   560  	}
   561  	for _, h := range goSum.m[mod] {
   562  		if !strings.HasPrefix(h, "h1:") {
   563  			continue
   564  		}
   565  		if !goSum.status[modSum{mod, h}].dirty {
   566  			return true
   567  		}
   568  	}
   569  	return false
   570  }
   571  
   572  // checkMod checks the given module's checksum and Go version.
   573  func checkMod(ctx context.Context, mod module.Version) {
   574  	// Do the file I/O before acquiring the go.sum lock.
   575  	ziphash, err := CachePath(ctx, mod, "ziphash")
   576  	if err != nil {
   577  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   578  	}
   579  	data, err := lockedfile.Read(ziphash)
   580  	if err != nil {
   581  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   582  	}
   583  	data = bytes.TrimSpace(data)
   584  	if !isValidSum(data) {
   585  		// Recreate ziphash file from zip file and use that to check the mod sum.
   586  		zip, err := CachePath(ctx, mod, "zip")
   587  		if err != nil {
   588  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   589  		}
   590  		err = hashZip(mod, zip, ziphash)
   591  		if err != nil {
   592  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   593  		}
   594  		return
   595  	}
   596  	h := string(data)
   597  	if !strings.HasPrefix(h, "h1:") {
   598  		base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h)))
   599  	}
   600  
   601  	if err := checkModSum(mod, h); err != nil {
   602  		base.Fatalf("%s", err)
   603  	}
   604  }
   605  
   606  // goModSum returns the checksum for the go.mod contents.
   607  func goModSum(data []byte) (string, error) {
   608  	return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
   609  		return io.NopCloser(bytes.NewReader(data)), nil
   610  	})
   611  }
   612  
   613  // checkGoMod checks the given module's go.mod checksum;
   614  // data is the go.mod content.
   615  func checkGoMod(path, version string, data []byte) error {
   616  	h, err := goModSum(data)
   617  	if err != nil {
   618  		return &module.ModuleError{Path: path, Version: version, Err: fmt.Errorf("verifying go.mod: %v", err)}
   619  	}
   620  
   621  	return checkModSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
   622  }
   623  
   624  // checkModSum checks that the recorded checksum for mod is h.
   625  //
   626  // mod.Version may have the additional suffix "/go.mod" to request the checksum
   627  // for the module's go.mod file only.
   628  func checkModSum(mod module.Version, h string) error {
   629  	// We lock goSum when manipulating it,
   630  	// but we arrange to release the lock when calling checkSumDB,
   631  	// so that parallel calls to checkModHash can execute parallel calls
   632  	// to checkSumDB.
   633  
   634  	// Check whether mod+h is listed in go.sum already. If so, we're done.
   635  	goSum.mu.Lock()
   636  	inited, err := initGoSum()
   637  	if err != nil {
   638  		goSum.mu.Unlock()
   639  		return err
   640  	}
   641  	done := inited && haveModSumLocked(mod, h)
   642  	if inited {
   643  		st := goSum.status[modSum{mod, h}]
   644  		st.used = true
   645  		goSum.status[modSum{mod, h}] = st
   646  	}
   647  	goSum.mu.Unlock()
   648  
   649  	if done {
   650  		return nil
   651  	}
   652  
   653  	// Not listed, so we want to add them.
   654  	// Consult checksum database if appropriate.
   655  	if useSumDB(mod) {
   656  		// Calls base.Fatalf if mismatch detected.
   657  		if err := checkSumDB(mod, h); err != nil {
   658  			return err
   659  		}
   660  	}
   661  
   662  	// Add mod+h to go.sum, if it hasn't appeared already.
   663  	if inited {
   664  		goSum.mu.Lock()
   665  		addModSumLocked(mod, h)
   666  		st := goSum.status[modSum{mod, h}]
   667  		st.dirty = true
   668  		goSum.status[modSum{mod, h}] = st
   669  		goSum.mu.Unlock()
   670  	}
   671  	return nil
   672  }
   673  
   674  // haveModSumLocked reports whether the pair mod,h is already listed in go.sum.
   675  // If it finds a conflicting pair instead, it calls base.Fatalf.
   676  // goSum.mu must be locked.
   677  func haveModSumLocked(mod module.Version, h string) bool {
   678  	sumFileName := "go.sum"
   679  	if strings.HasSuffix(GoSumFile, "go.work.sum") {
   680  		sumFileName = "go.work.sum"
   681  	}
   682  	for _, vh := range goSum.m[mod] {
   683  		if h == vh {
   684  			return true
   685  		}
   686  		if strings.HasPrefix(vh, "h1:") {
   687  			base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, sumFileName, vh)
   688  		}
   689  	}
   690  	// Also check workspace sums.
   691  	foundMatch := false
   692  	// Check sums from all files in case there are conflicts between
   693  	// the files.
   694  	for goSumFile, goSums := range goSum.w {
   695  		for _, vh := range goSums[mod] {
   696  			if h == vh {
   697  				foundMatch = true
   698  			} else if strings.HasPrefix(vh, "h1:") {
   699  				base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, goSumFile, vh)
   700  			}
   701  		}
   702  	}
   703  	return foundMatch
   704  }
   705  
   706  // addModSumLocked adds the pair mod,h to go.sum.
   707  // goSum.mu must be locked.
   708  func addModSumLocked(mod module.Version, h string) {
   709  	if haveModSumLocked(mod, h) {
   710  		return
   711  	}
   712  	if len(goSum.m[mod]) > 0 {
   713  		fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h)
   714  	}
   715  	goSum.m[mod] = append(goSum.m[mod], h)
   716  }
   717  
   718  // checkSumDB checks the mod, h pair against the Go checksum database.
   719  // It calls base.Fatalf if the hash is to be rejected.
   720  func checkSumDB(mod module.Version, h string) error {
   721  	modWithoutSuffix := mod
   722  	noun := "module"
   723  	if before, found := strings.CutSuffix(mod.Version, "/go.mod"); found {
   724  		noun = "go.mod"
   725  		modWithoutSuffix.Version = before
   726  	}
   727  
   728  	db, lines, err := lookupSumDB(mod)
   729  	if err != nil {
   730  		return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: %v", noun, err))
   731  	}
   732  
   733  	have := mod.Path + " " + mod.Version + " " + h
   734  	prefix := mod.Path + " " + mod.Version + " h1:"
   735  	for _, line := range lines {
   736  		if line == have {
   737  			return nil
   738  		}
   739  		if strings.HasPrefix(line, prefix) {
   740  			return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, noun, h, db, line[len(prefix)-len("h1:"):]))
   741  		}
   742  	}
   743  	return nil
   744  }
   745  
   746  // Sum returns the checksum for the downloaded copy of the given module,
   747  // if present in the download cache.
   748  func Sum(ctx context.Context, mod module.Version) string {
   749  	if cfg.GOMODCACHE == "" {
   750  		// Do not use current directory.
   751  		return ""
   752  	}
   753  
   754  	ziphash, err := CachePath(ctx, mod, "ziphash")
   755  	if err != nil {
   756  		return ""
   757  	}
   758  	data, err := lockedfile.Read(ziphash)
   759  	if err != nil {
   760  		return ""
   761  	}
   762  	data = bytes.TrimSpace(data)
   763  	if !isValidSum(data) {
   764  		return ""
   765  	}
   766  	return string(data)
   767  }
   768  
   769  // isValidSum returns true if data is the valid contents of a zip hash file.
   770  // Certain critical files are written to disk by first truncating
   771  // then writing the actual bytes, so that if the write fails
   772  // the corrupt file should contain at least one of the null
   773  // bytes written by the truncate operation.
   774  func isValidSum(data []byte) bool {
   775  	if bytes.IndexByte(data, '\000') >= 0 {
   776  		return false
   777  	}
   778  
   779  	if len(data) != len("h1:")+base64.StdEncoding.EncodedLen(sha256.Size) {
   780  		return false
   781  	}
   782  
   783  	return true
   784  }
   785  
   786  var ErrGoSumDirty = errors.New("updates to go.sum needed, disabled by -mod=readonly")
   787  
   788  // WriteGoSum writes the go.sum file if it needs to be updated.
   789  //
   790  // keep is used to check whether a newly added sum should be saved in go.sum.
   791  // It should have entries for both module content sums and go.mod sums
   792  // (version ends with "/go.mod"). Existing sums will be preserved unless they
   793  // have been marked for deletion with TrimGoSum.
   794  func WriteGoSum(ctx context.Context, keep map[module.Version]bool, readonly bool) error {
   795  	goSum.mu.Lock()
   796  	defer goSum.mu.Unlock()
   797  
   798  	// If we haven't read the go.sum file yet, don't bother writing it.
   799  	if !goSum.enabled {
   800  		return nil
   801  	}
   802  
   803  	// Check whether we need to add sums for which keep[m] is true or remove
   804  	// unused sums marked with TrimGoSum. If there are no changes to make,
   805  	// just return without opening go.sum.
   806  	dirty := false
   807  Outer:
   808  	for m, hs := range goSum.m {
   809  		for _, h := range hs {
   810  			st := goSum.status[modSum{m, h}]
   811  			if st.dirty && (!st.used || keep[m]) {
   812  				dirty = true
   813  				break Outer
   814  			}
   815  		}
   816  	}
   817  	if !dirty {
   818  		return nil
   819  	}
   820  	if readonly {
   821  		return ErrGoSumDirty
   822  	}
   823  	if _, ok := fsys.OverlayPath(GoSumFile); ok {
   824  		base.Fatalf("go: updates to go.sum needed, but go.sum is part of the overlay specified with -overlay")
   825  	}
   826  
   827  	// Make a best-effort attempt to acquire the side lock, only to exclude
   828  	// previous versions of the 'go' command from making simultaneous edits.
   829  	if unlock, err := SideLock(ctx); err == nil {
   830  		defer unlock()
   831  	}
   832  
   833  	err := lockedfile.Transform(GoSumFile, func(data []byte) ([]byte, error) {
   834  		if !goSum.overwrite {
   835  			// Incorporate any sums added by other processes in the meantime.
   836  			// Add only the sums that we actually checked: the user may have edited or
   837  			// truncated the file to remove erroneous hashes, and we shouldn't restore
   838  			// them without good reason.
   839  			goSum.m = make(map[module.Version][]string, len(goSum.m))
   840  			readGoSum(goSum.m, GoSumFile, data)
   841  			for ms, st := range goSum.status {
   842  				if st.used && !sumInWorkspaceModulesLocked(ms.mod) {
   843  					addModSumLocked(ms.mod, ms.sum)
   844  				}
   845  			}
   846  		}
   847  
   848  		var mods []module.Version
   849  		for m := range goSum.m {
   850  			mods = append(mods, m)
   851  		}
   852  		module.Sort(mods)
   853  
   854  		var buf bytes.Buffer
   855  		for _, m := range mods {
   856  			list := goSum.m[m]
   857  			sort.Strings(list)
   858  			str.Uniq(&list)
   859  			for _, h := range list {
   860  				st := goSum.status[modSum{m, h}]
   861  				if (!st.dirty || (st.used && keep[m])) && !sumInWorkspaceModulesLocked(m) {
   862  					fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
   863  				}
   864  			}
   865  		}
   866  		return buf.Bytes(), nil
   867  	})
   868  
   869  	if err != nil {
   870  		return fmt.Errorf("updating go.sum: %w", err)
   871  	}
   872  
   873  	goSum.status = make(map[modSum]modSumStatus)
   874  	goSum.overwrite = false
   875  	return nil
   876  }
   877  
   878  func sumInWorkspaceModulesLocked(m module.Version) bool {
   879  	for _, goSums := range goSum.w {
   880  		if _, ok := goSums[m]; ok {
   881  			return true
   882  		}
   883  	}
   884  	return false
   885  }
   886  
   887  // TrimGoSum trims go.sum to contain only the modules needed for reproducible
   888  // builds.
   889  //
   890  // keep is used to check whether a sum should be retained in go.mod. It should
   891  // have entries for both module content sums and go.mod sums (version ends
   892  // with "/go.mod").
   893  func TrimGoSum(keep map[module.Version]bool) {
   894  	goSum.mu.Lock()
   895  	defer goSum.mu.Unlock()
   896  	inited, err := initGoSum()
   897  	if err != nil {
   898  		base.Fatalf("%s", err)
   899  	}
   900  	if !inited {
   901  		return
   902  	}
   903  
   904  	for m, hs := range goSum.m {
   905  		if !keep[m] {
   906  			for _, h := range hs {
   907  				goSum.status[modSum{m, h}] = modSumStatus{used: false, dirty: true}
   908  			}
   909  			goSum.overwrite = true
   910  		}
   911  	}
   912  }
   913  
   914  const goSumMismatch = `
   915  
   916  SECURITY ERROR
   917  This download does NOT match an earlier download recorded in go.sum.
   918  The bits may have been replaced on the origin server, or an attacker may
   919  have intercepted the download attempt.
   920  
   921  For more information, see 'go help module-auth'.
   922  `
   923  
   924  const sumdbMismatch = `
   925  
   926  SECURITY ERROR
   927  This download does NOT match the one reported by the checksum server.
   928  The bits may have been replaced on the origin server, or an attacker may
   929  have intercepted the download attempt.
   930  
   931  For more information, see 'go help module-auth'.
   932  `
   933  
   934  const hashVersionMismatch = `
   935  
   936  SECURITY WARNING
   937  This download is listed in go.sum, but using an unknown hash algorithm.
   938  The download cannot be verified.
   939  
   940  For more information, see 'go help module-auth'.
   941  
   942  `
   943  
   944  var HelpModuleAuth = &base.Command{
   945  	UsageLine: "module-auth",
   946  	Short:     "module authentication using go.sum",
   947  	Long: `
   948  When the go command downloads a module zip file or go.mod file into the
   949  module cache, it computes a cryptographic hash and compares it with a known
   950  value to verify the file hasn't changed since it was first downloaded. Known
   951  hashes are stored in a file in the module root directory named go.sum. Hashes
   952  may also be downloaded from the checksum database depending on the values of
   953  GOSUMDB, GOPRIVATE, and GONOSUMDB.
   954  
   955  For details, see https://golang.org/ref/mod#authenticating.
   956  `,
   957  }
   958  
   959  var HelpPrivate = &base.Command{
   960  	UsageLine: "private",
   961  	Short:     "configuration for downloading non-public code",
   962  	Long: `
   963  The go command defaults to downloading modules from the public Go module
   964  mirror at proxy.golang.org. It also defaults to validating downloaded modules,
   965  regardless of source, against the public Go checksum database at sum.golang.org.
   966  These defaults work well for publicly available source code.
   967  
   968  The GOPRIVATE environment variable controls which modules the go command
   969  considers to be private (not available publicly) and should therefore not use
   970  the proxy or checksum database. The variable is a comma-separated list of
   971  glob patterns (in the syntax of Go's path.Match) of module path prefixes.
   972  For example,
   973  
   974  	GOPRIVATE=*.corp.example.com,rsc.io/private
   975  
   976  causes the go command to treat as private any module with a path prefix
   977  matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
   978  and rsc.io/private/quux.
   979  
   980  For fine-grained control over module download and validation, the GONOPROXY
   981  and GONOSUMDB environment variables accept the same kind of glob list
   982  and override GOPRIVATE for the specific decision of whether to use the proxy
   983  and checksum database, respectively.
   984  
   985  For example, if a company ran a module proxy serving private modules,
   986  users would configure go using:
   987  
   988  	GOPRIVATE=*.corp.example.com
   989  	GOPROXY=proxy.example.com
   990  	GONOPROXY=none
   991  
   992  The GOPRIVATE variable is also used to define the "public" and "private"
   993  patterns for the GOVCS variable; see 'go help vcs'. For that usage,
   994  GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths
   995  instead of module paths.
   996  
   997  The 'go env -w' command (see 'go help env') can be used to set these variables
   998  for future go command invocations.
   999  
  1000  For more details, see https://golang.org/ref/mod#private-modules.
  1001  `,
  1002  }
  1003  

View as plain text