cache.go

Documentation: cmd/go/internal/modfetch

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"encoding/json"
    11  	"errors"
    12  	"fmt"
    13  	"io"
    14  	"io/fs"
    15  	"math/rand"
    16  	"os"
    17  	"path/filepath"
    18  	"strconv"
    19  	"strings"
    20  	"sync"
    21  
    22  	"cmd/go/internal/base"
    23  	"cmd/go/internal/cfg"
    24  	"cmd/go/internal/gover"
    25  	"cmd/go/internal/lockedfile"
    26  	"cmd/go/internal/modfetch/codehost"
    27  	"cmd/go/internal/par"
    28  	"cmd/go/internal/robustio"
    29  
    30  	"golang.org/x/mod/module"
    31  	"golang.org/x/mod/semver"
    32  )
    33  
    34  func cacheDir(ctx context.Context, path string) (string, error) {
    35  	if err := checkCacheDir(ctx); err != nil {
    36  		return "", err
    37  	}
    38  	enc, err := module.EscapePath(path)
    39  	if err != nil {
    40  		return "", err
    41  	}
    42  	return filepath.Join(cfg.GOMODCACHE, "cache/download", enc, "/@v"), nil
    43  }
    44  
    45  func CachePath(ctx context.Context, m module.Version, suffix string) (string, error) {
    46  	if gover.IsToolchain(m.Path) {
    47  		return "", ErrToolchain
    48  	}
    49  	dir, err := cacheDir(ctx, m.Path)
    50  	if err != nil {
    51  		return "", err
    52  	}
    53  	if !gover.ModIsValid(m.Path, m.Version) {
    54  		return "", fmt.Errorf("non-semver module version %q", m.Version)
    55  	}
    56  	if module.CanonicalVersion(m.Version) != m.Version {
    57  		return "", fmt.Errorf("non-canonical module version %q", m.Version)
    58  	}
    59  	encVer, err := module.EscapeVersion(m.Version)
    60  	if err != nil {
    61  		return "", err
    62  	}
    63  	return filepath.Join(dir, encVer+"."+suffix), nil
    64  }
    65  
    66  // DownloadDir returns the directory to which m should have been downloaded.
    67  // An error will be returned if the module path or version cannot be escaped.
    68  // An error satisfying errors.Is(err, fs.ErrNotExist) will be returned
    69  // along with the directory if the directory does not exist or if the directory
    70  // is not completely populated.
    71  func DownloadDir(ctx context.Context, m module.Version) (string, error) {
    72  	if gover.IsToolchain(m.Path) {
    73  		return "", ErrToolchain
    74  	}
    75  	if err := checkCacheDir(ctx); err != nil {
    76  		return "", err
    77  	}
    78  	enc, err := module.EscapePath(m.Path)
    79  	if err != nil {
    80  		return "", err
    81  	}
    82  	if !gover.ModIsValid(m.Path, m.Version) {
    83  		return "", fmt.Errorf("non-semver module version %q", m.Version)
    84  	}
    85  	if module.CanonicalVersion(m.Version) != m.Version {
    86  		return "", fmt.Errorf("non-canonical module version %q", m.Version)
    87  	}
    88  	encVer, err := module.EscapeVersion(m.Version)
    89  	if err != nil {
    90  		return "", err
    91  	}
    92  
    93  	// Check whether the directory itself exists.
    94  	dir := filepath.Join(cfg.GOMODCACHE, enc+"@"+encVer)
    95  	if fi, err := os.Stat(dir); os.IsNotExist(err) {
    96  		return dir, err
    97  	} else if err != nil {
    98  		return dir, &DownloadDirPartialError{dir, err}
    99  	} else if !fi.IsDir() {
   100  		return dir, &DownloadDirPartialError{dir, errors.New("not a directory")}
   101  	}
   102  
   103  	// Check if a .partial file exists. This is created at the beginning of
   104  	// a download and removed after the zip is extracted.
   105  	partialPath, err := CachePath(ctx, m, "partial")
   106  	if err != nil {
   107  		return dir, err
   108  	}
   109  	if _, err := os.Stat(partialPath); err == nil {
   110  		return dir, &DownloadDirPartialError{dir, errors.New("not completely extracted")}
   111  	} else if !os.IsNotExist(err) {
   112  		return dir, err
   113  	}
   114  
   115  	// Check if a .ziphash file exists. It should be created before the
   116  	// zip is extracted, but if it was deleted (by another program?), we need
   117  	// to re-calculate it. Note that checkMod will repopulate the ziphash
   118  	// file if it doesn't exist, but if the module is excluded by checks
   119  	// through GONOSUMDB or GOPRIVATE, that check and repopulation won't happen.
   120  	ziphashPath, err := CachePath(ctx, m, "ziphash")
   121  	if err != nil {
   122  		return dir, err
   123  	}
   124  	if _, err := os.Stat(ziphashPath); os.IsNotExist(err) {
   125  		return dir, &DownloadDirPartialError{dir, errors.New("ziphash file is missing")}
   126  	} else if err != nil {
   127  		return dir, err
   128  	}
   129  	return dir, nil
   130  }
   131  
   132  // DownloadDirPartialError is returned by DownloadDir if a module directory
   133  // exists but was not completely populated.
   134  //
   135  // DownloadDirPartialError is equivalent to fs.ErrNotExist.
   136  type DownloadDirPartialError struct {
   137  	Dir string
   138  	Err error
   139  }
   140  
   141  func (e *DownloadDirPartialError) Error() string     { return fmt.Sprintf("%s: %v", e.Dir, e.Err) }
   142  func (e *DownloadDirPartialError) Is(err error) bool { return err == fs.ErrNotExist }
   143  
   144  // lockVersion locks a file within the module cache that guards the downloading
   145  // and extraction of the zipfile for the given module version.
   146  func lockVersion(ctx context.Context, mod module.Version) (unlock func(), err error) {
   147  	path, err := CachePath(ctx, mod, "lock")
   148  	if err != nil {
   149  		return nil, err
   150  	}
   151  	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
   152  		return nil, err
   153  	}
   154  	return lockedfile.MutexAt(path).Lock()
   155  }
   156  
   157  // SideLock locks a file within the module cache that previously guarded
   158  // edits to files outside the cache, such as go.sum and go.mod files in the
   159  // user's working directory.
   160  // If err is nil, the caller MUST eventually call the unlock function.
   161  func SideLock(ctx context.Context) (unlock func(), err error) {
   162  	if err := checkCacheDir(ctx); err != nil {
   163  		return nil, err
   164  	}
   165  
   166  	path := filepath.Join(cfg.GOMODCACHE, "cache", "lock")
   167  	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
   168  		return nil, fmt.Errorf("failed to create cache directory: %w", err)
   169  	}
   170  
   171  	return lockedfile.MutexAt(path).Lock()
   172  }
   173  
   174  // A cachingRepo is a cache around an underlying Repo,
   175  // avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not CheckReuse or Zip).
   176  // It is also safe for simultaneous use by multiple goroutines
   177  // (so that it can be returned from Lookup multiple times).
   178  // It serializes calls to the underlying Repo.
   179  type cachingRepo struct {
   180  	path          string
   181  	versionsCache par.ErrCache[string, *Versions]
   182  	statCache     par.ErrCache[string, *RevInfo]
   183  	latestCache   par.ErrCache[struct{}, *RevInfo]
   184  	gomodCache    par.ErrCache[string, []byte]
   185  
   186  	once     sync.Once
   187  	initRepo func(context.Context) (Repo, error)
   188  	r        Repo
   189  }
   190  
   191  func newCachingRepo(ctx context.Context, path string, initRepo func(context.Context) (Repo, error)) *cachingRepo {
   192  	return &cachingRepo{
   193  		path:     path,
   194  		initRepo: initRepo,
   195  	}
   196  }
   197  
   198  func (r *cachingRepo) repo(ctx context.Context) Repo {
   199  	r.once.Do(func() {
   200  		var err error
   201  		r.r, err = r.initRepo(ctx)
   202  		if err != nil {
   203  			r.r = errRepo{r.path, err}
   204  		}
   205  	})
   206  	return r.r
   207  }
   208  
   209  func (r *cachingRepo) CheckReuse(ctx context.Context, old *codehost.Origin) error {
   210  	return r.repo(ctx).CheckReuse(ctx, old)
   211  }
   212  
   213  func (r *cachingRepo) ModulePath() string {
   214  	return r.path
   215  }
   216  
   217  func (r *cachingRepo) Versions(ctx context.Context, prefix string) (*Versions, error) {
   218  	v, err := r.versionsCache.Do(prefix, func() (*Versions, error) {
   219  		return r.repo(ctx).Versions(ctx, prefix)
   220  	})
   221  
   222  	if err != nil {
   223  		return nil, err
   224  	}
   225  	return &Versions{
   226  		Origin: v.Origin,
   227  		List:   append([]string(nil), v.List...),
   228  	}, nil
   229  }
   230  
   231  type cachedInfo struct {
   232  	info *RevInfo
   233  	err  error
   234  }
   235  
   236  func (r *cachingRepo) Stat(ctx context.Context, rev string) (*RevInfo, error) {
   237  	if gover.IsToolchain(r.path) {
   238  		// Skip disk cache; the underlying golang.org/toolchain repo is cached instead.
   239  		return r.repo(ctx).Stat(ctx, rev)
   240  	}
   241  	info, err := r.statCache.Do(rev, func() (*RevInfo, error) {
   242  		file, info, err := readDiskStat(ctx, r.path, rev)
   243  		if err == nil {
   244  			return info, err
   245  		}
   246  
   247  		info, err = r.repo(ctx).Stat(ctx, rev)
   248  		if err == nil {
   249  			// If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78,
   250  			// then save the information under the proper version, for future use.
   251  			if info.Version != rev {
   252  				file, _ = CachePath(ctx, module.Version{Path: r.path, Version: info.Version}, "info")
   253  				r.statCache.Do(info.Version, func() (*RevInfo, error) {
   254  					return info, nil
   255  				})
   256  			}
   257  
   258  			if err := writeDiskStat(ctx, file, info); err != nil {
   259  				fmt.Fprintf(os.Stderr, "go: writing stat cache: %v\n", err)
   260  			}
   261  		}
   262  		return info, err
   263  	})
   264  	if info != nil {
   265  		copy := *info
   266  		info = &copy
   267  	}
   268  	return info, err
   269  }
   270  
   271  func (r *cachingRepo) Latest(ctx context.Context) (*RevInfo, error) {
   272  	if gover.IsToolchain(r.path) {
   273  		// Skip disk cache; the underlying golang.org/toolchain repo is cached instead.
   274  		return r.repo(ctx).Latest(ctx)
   275  	}
   276  	info, err := r.latestCache.Do(struct{}{}, func() (*RevInfo, error) {
   277  		info, err := r.repo(ctx).Latest(ctx)
   278  
   279  		// Save info for likely future Stat call.
   280  		if err == nil {
   281  			r.statCache.Do(info.Version, func() (*RevInfo, error) {
   282  				return info, nil
   283  			})
   284  			if file, _, err := readDiskStat(ctx, r.path, info.Version); err != nil {
   285  				writeDiskStat(ctx, file, info)
   286  			}
   287  		}
   288  
   289  		return info, err
   290  	})
   291  	if info != nil {
   292  		copy := *info
   293  		info = &copy
   294  	}
   295  	return info, err
   296  }
   297  
   298  func (r *cachingRepo) GoMod(ctx context.Context, version string) ([]byte, error) {
   299  	if gover.IsToolchain(r.path) {
   300  		// Skip disk cache; the underlying golang.org/toolchain repo is cached instead.
   301  		return r.repo(ctx).GoMod(ctx, version)
   302  	}
   303  	text, err := r.gomodCache.Do(version, func() ([]byte, error) {
   304  		file, text, err := readDiskGoMod(ctx, r.path, version)
   305  		if err == nil {
   306  			// Note: readDiskGoMod already called checkGoMod.
   307  			return text, nil
   308  		}
   309  
   310  		text, err = r.repo(ctx).GoMod(ctx, version)
   311  		if err == nil {
   312  			if err := checkGoMod(r.path, version, text); err != nil {
   313  				return text, err
   314  			}
   315  			if err := writeDiskGoMod(ctx, file, text); err != nil {
   316  				fmt.Fprintf(os.Stderr, "go: writing go.mod cache: %v\n", err)
   317  			}
   318  		}
   319  		return text, err
   320  	})
   321  	if err != nil {
   322  		return nil, err
   323  	}
   324  	return append([]byte(nil), text...), nil
   325  }
   326  
   327  func (r *cachingRepo) Zip(ctx context.Context, dst io.Writer, version string) error {
   328  	if gover.IsToolchain(r.path) {
   329  		return ErrToolchain
   330  	}
   331  	return r.repo(ctx).Zip(ctx, dst, version)
   332  }
   333  
   334  // InfoFile is like Lookup(ctx, path).Stat(version) but also returns the name of the file
   335  // containing the cached information.
   336  func InfoFile(ctx context.Context, path, version string) (*RevInfo, string, error) {
   337  	if !gover.ModIsValid(path, version) {
   338  		return nil, "", fmt.Errorf("invalid version %q", version)
   339  	}
   340  
   341  	if file, info, err := readDiskStat(ctx, path, version); err == nil {
   342  		return info, file, nil
   343  	}
   344  
   345  	var info *RevInfo
   346  	var err2info map[error]*RevInfo
   347  	err := TryProxies(func(proxy string) error {
   348  		i, err := Lookup(ctx, proxy, path).Stat(ctx, version)
   349  		if err == nil {
   350  			info = i
   351  		} else {
   352  			if err2info == nil {
   353  				err2info = make(map[error]*RevInfo)
   354  			}
   355  			err2info[err] = info
   356  		}
   357  		return err
   358  	})
   359  	if err != nil {
   360  		return err2info[err], "", err
   361  	}
   362  
   363  	// Stat should have populated the disk cache for us.
   364  	file, err := CachePath(ctx, module.Version{Path: path, Version: version}, "info")
   365  	if err != nil {
   366  		return nil, "", err
   367  	}
   368  	return info, file, nil
   369  }
   370  
   371  // GoMod is like Lookup(ctx, path).GoMod(rev) but avoids the
   372  // repository path resolution in Lookup if the result is
   373  // already cached on local disk.
   374  func GoMod(ctx context.Context, path, rev string) ([]byte, error) {
   375  	// Convert commit hash to pseudo-version
   376  	// to increase cache hit rate.
   377  	if !gover.ModIsValid(path, rev) {
   378  		if _, info, err := readDiskStat(ctx, path, rev); err == nil {
   379  			rev = info.Version
   380  		} else {
   381  			if errors.Is(err, statCacheErr) {
   382  				return nil, err
   383  			}
   384  			err := TryProxies(func(proxy string) error {
   385  				info, err := Lookup(ctx, proxy, path).Stat(ctx, rev)
   386  				if err == nil {
   387  					rev = info.Version
   388  				}
   389  				return err
   390  			})
   391  			if err != nil {
   392  				return nil, err
   393  			}
   394  		}
   395  	}
   396  
   397  	_, data, err := readDiskGoMod(ctx, path, rev)
   398  	if err == nil {
   399  		return data, nil
   400  	}
   401  
   402  	err = TryProxies(func(proxy string) (err error) {
   403  		data, err = Lookup(ctx, proxy, path).GoMod(ctx, rev)
   404  		return err
   405  	})
   406  	return data, err
   407  }
   408  
   409  // GoModFile is like GoMod but returns the name of the file containing
   410  // the cached information.
   411  func GoModFile(ctx context.Context, path, version string) (string, error) {
   412  	if !gover.ModIsValid(path, version) {
   413  		return "", fmt.Errorf("invalid version %q", version)
   414  	}
   415  	if _, err := GoMod(ctx, path, version); err != nil {
   416  		return "", err
   417  	}
   418  	// GoMod should have populated the disk cache for us.
   419  	file, err := CachePath(ctx, module.Version{Path: path, Version: version}, "mod")
   420  	if err != nil {
   421  		return "", err
   422  	}
   423  	return file, nil
   424  }
   425  
   426  // GoModSum returns the go.sum entry for the module version's go.mod file.
   427  // (That is, it returns the entry listed in go.sum as "path version/go.mod".)
   428  func GoModSum(ctx context.Context, path, version string) (string, error) {
   429  	if !gover.ModIsValid(path, version) {
   430  		return "", fmt.Errorf("invalid version %q", version)
   431  	}
   432  	data, err := GoMod(ctx, path, version)
   433  	if err != nil {
   434  		return "", err
   435  	}
   436  	sum, err := goModSum(data)
   437  	if err != nil {
   438  		return "", err
   439  	}
   440  	return sum, nil
   441  }
   442  
   443  var errNotCached = fmt.Errorf("not in cache")
   444  
   445  // readDiskStat reads a cached stat result from disk,
   446  // returning the name of the cache file and the result.
   447  // If the read fails, the caller can use
   448  // writeDiskStat(file, info) to write a new cache entry.
   449  func readDiskStat(ctx context.Context, path, rev string) (file string, info *RevInfo, err error) {
   450  	if gover.IsToolchain(path) {
   451  		return "", nil, errNotCached
   452  	}
   453  	file, data, err := readDiskCache(ctx, path, rev, "info")
   454  	if err != nil {
   455  		// If the cache already contains a pseudo-version with the given hash, we
   456  		// would previously return that pseudo-version without checking upstream.
   457  		// However, that produced an unfortunate side-effect: if the author added a
   458  		// tag to the repository, 'go get' would not pick up the effect of that new
   459  		// tag on the existing commits, and 'go' commands that referred to those
   460  		// commits would use the previous name instead of the new one.
   461  		//
   462  		// That's especially problematic if the original pseudo-version starts with
   463  		// v0.0.0-, as was the case for all pseudo-versions during vgo development,
   464  		// since a v0.0.0- pseudo-version has lower precedence than pretty much any
   465  		// tagged version.
   466  		//
   467  		// In practice, we're only looking up by hash during initial conversion of a
   468  		// legacy config and during an explicit 'go get', and a little extra latency
   469  		// for those operations seems worth the benefit of picking up more accurate
   470  		// versions.
   471  		//
   472  		// Fall back to this resolution scheme only if the GOPROXY setting prohibits
   473  		// us from resolving upstream tags.
   474  		if cfg.GOPROXY == "off" {
   475  			if file, info, err := readDiskStatByHash(ctx, path, rev); err == nil {
   476  				return file, info, nil
   477  			}
   478  		}
   479  		return file, nil, err
   480  	}
   481  	info = new(RevInfo)
   482  	if err := json.Unmarshal(data, info); err != nil {
   483  		return file, nil, errNotCached
   484  	}
   485  	// The disk might have stale .info files that have Name and Short fields set.
   486  	// We want to canonicalize to .info files with those fields omitted.
   487  	// Remarshal and update the cache file if needed.
   488  	data2, err := json.Marshal(info)
   489  	if err == nil && !bytes.Equal(data2, data) {
   490  		writeDiskCache(ctx, file, data)
   491  	}
   492  	return file, info, nil
   493  }
   494  
   495  // readDiskStatByHash is a fallback for readDiskStat for the case
   496  // where rev is a commit hash instead of a proper semantic version.
   497  // In that case, we look for a cached pseudo-version that matches
   498  // the commit hash. If we find one, we use it.
   499  // This matters most for converting legacy package management
   500  // configs, when we are often looking up commits by full hash.
   501  // Without this check we'd be doing network I/O to the remote repo
   502  // just to find out about a commit we already know about
   503  // (and have cached under its pseudo-version).
   504  func readDiskStatByHash(ctx context.Context, path, rev string) (file string, info *RevInfo, err error) {
   505  	if gover.IsToolchain(path) {
   506  		return "", nil, errNotCached
   507  	}
   508  	if cfg.GOMODCACHE == "" {
   509  		// Do not download to current directory.
   510  		return "", nil, errNotCached
   511  	}
   512  
   513  	if !codehost.AllHex(rev) || len(rev) < 12 {
   514  		return "", nil, errNotCached
   515  	}
   516  	rev = rev[:12]
   517  	cdir, err := cacheDir(ctx, path)
   518  	if err != nil {
   519  		return "", nil, errNotCached
   520  	}
   521  	dir, err := os.Open(cdir)
   522  	if err != nil {
   523  		return "", nil, errNotCached
   524  	}
   525  	names, err := dir.Readdirnames(-1)
   526  	dir.Close()
   527  	if err != nil {
   528  		return "", nil, errNotCached
   529  	}
   530  
   531  	// A given commit hash may map to more than one pseudo-version,
   532  	// depending on which tags are present on the repository.
   533  	// Take the highest such version.
   534  	var maxVersion string
   535  	suffix := "-" + rev + ".info"
   536  	err = errNotCached
   537  	for _, name := range names {
   538  		if strings.HasSuffix(name, suffix) {
   539  			v := strings.TrimSuffix(name, ".info")
   540  			if module.IsPseudoVersion(v) && semver.Compare(v, maxVersion) > 0 {
   541  				maxVersion = v
   542  				file, info, err = readDiskStat(ctx, path, strings.TrimSuffix(name, ".info"))
   543  			}
   544  		}
   545  	}
   546  	return file, info, err
   547  }
   548  
   549  // oldVgoPrefix is the prefix in the old auto-generated cached go.mod files.
   550  // We stopped trying to auto-generate the go.mod files. Now we use a trivial
   551  // go.mod with only a module line, and we've dropped the version prefix
   552  // entirely. If we see a version prefix, that means we're looking at an old copy
   553  // and should ignore it.
   554  var oldVgoPrefix = []byte("//vgo 0.0.")
   555  
   556  // readDiskGoMod reads a cached go.mod file from disk,
   557  // returning the name of the cache file and the result.
   558  // If the read fails, the caller can use
   559  // writeDiskGoMod(file, data) to write a new cache entry.
   560  func readDiskGoMod(ctx context.Context, path, rev string) (file string, data []byte, err error) {
   561  	if gover.IsToolchain(path) {
   562  		return "", nil, errNotCached
   563  	}
   564  	file, data, err = readDiskCache(ctx, path, rev, "mod")
   565  
   566  	// If the file has an old auto-conversion prefix, pretend it's not there.
   567  	if bytes.HasPrefix(data, oldVgoPrefix) {
   568  		err = errNotCached
   569  		data = nil
   570  	}
   571  
   572  	if err == nil {
   573  		if err := checkGoMod(path, rev, data); err != nil {
   574  			return "", nil, err
   575  		}
   576  	}
   577  
   578  	return file, data, err
   579  }
   580  
   581  // readDiskCache is the generic "read from a cache file" implementation.
   582  // It takes the revision and an identifying suffix for the kind of data being cached.
   583  // It returns the name of the cache file and the content of the file.
   584  // If the read fails, the caller can use
   585  // writeDiskCache(file, data) to write a new cache entry.
   586  func readDiskCache(ctx context.Context, path, rev, suffix string) (file string, data []byte, err error) {
   587  	if gover.IsToolchain(path) {
   588  		return "", nil, errNotCached
   589  	}
   590  	file, err = CachePath(ctx, module.Version{Path: path, Version: rev}, suffix)
   591  	if err != nil {
   592  		return "", nil, errNotCached
   593  	}
   594  	data, err = robustio.ReadFile(file)
   595  	if err != nil {
   596  		return file, nil, errNotCached
   597  	}
   598  	return file, data, nil
   599  }
   600  
   601  // writeDiskStat writes a stat result cache entry.
   602  // The file name must have been returned by a previous call to readDiskStat.
   603  func writeDiskStat(ctx context.Context, file string, info *RevInfo) error {
   604  	if file == "" {
   605  		return nil
   606  	}
   607  
   608  	if info.Origin != nil {
   609  		// Clean the origin information, which might have too many
   610  		// validation criteria, for example if we are saving the result of
   611  		// m@master as m@pseudo-version.
   612  		clean := *info
   613  		info = &clean
   614  		o := *info.Origin
   615  		info.Origin = &o
   616  
   617  		// Tags never matter if you are starting with a semver version,
   618  		// as we would be when finding this cache entry.
   619  		o.TagSum = ""
   620  		o.TagPrefix = ""
   621  		// Ref doesn't matter if you have a pseudoversion.
   622  		if module.IsPseudoVersion(info.Version) {
   623  			o.Ref = ""
   624  		}
   625  	}
   626  
   627  	js, err := json.Marshal(info)
   628  	if err != nil {
   629  		return err
   630  	}
   631  	return writeDiskCache(ctx, file, js)
   632  }
   633  
   634  // writeDiskGoMod writes a go.mod cache entry.
   635  // The file name must have been returned by a previous call to readDiskGoMod.
   636  func writeDiskGoMod(ctx context.Context, file string, text []byte) error {
   637  	return writeDiskCache(ctx, file, text)
   638  }
   639  
   640  // writeDiskCache is the generic "write to a cache file" implementation.
   641  // The file must have been returned by a previous call to readDiskCache.
   642  func writeDiskCache(ctx context.Context, file string, data []byte) error {
   643  	if file == "" {
   644  		return nil
   645  	}
   646  	// Make sure directory for file exists.
   647  	if err := os.MkdirAll(filepath.Dir(file), 0777); err != nil {
   648  		return err
   649  	}
   650  
   651  	// Write the file to a temporary location, and then rename it to its final
   652  	// path to reduce the likelihood of a corrupt file existing at that final path.
   653  	f, err := tempFile(ctx, filepath.Dir(file), filepath.Base(file), 0666)
   654  	if err != nil {
   655  		return err
   656  	}
   657  	defer func() {
   658  		// Only call os.Remove on f.Name() if we failed to rename it: otherwise,
   659  		// some other process may have created a new file with the same name after
   660  		// the rename completed.
   661  		if err != nil {
   662  			f.Close()
   663  			os.Remove(f.Name())
   664  		}
   665  	}()
   666  
   667  	if _, err := f.Write(data); err != nil {
   668  		return err
   669  	}
   670  	if err := f.Close(); err != nil {
   671  		return err
   672  	}
   673  	if err := robustio.Rename(f.Name(), file); err != nil {
   674  		return err
   675  	}
   676  
   677  	if strings.HasSuffix(file, ".mod") {
   678  		rewriteVersionList(ctx, filepath.Dir(file))
   679  	}
   680  	return nil
   681  }
   682  
   683  // tempFile creates a new temporary file with given permission bits.
   684  func tempFile(ctx context.Context, dir, prefix string, perm fs.FileMode) (f *os.File, err error) {
   685  	for i := 0; i < 10000; i++ {
   686  		name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+".tmp")
   687  		f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm)
   688  		if os.IsExist(err) {
   689  			if ctx.Err() != nil {
   690  				return nil, ctx.Err()
   691  			}
   692  			continue
   693  		}
   694  		break
   695  	}
   696  	return
   697  }
   698  
   699  // rewriteVersionList rewrites the version list in dir
   700  // after a new *.mod file has been written.
   701  func rewriteVersionList(ctx context.Context, dir string) (err error) {
   702  	if filepath.Base(dir) != "@v" {
   703  		base.Fatalf("go: internal error: misuse of rewriteVersionList")
   704  	}
   705  
   706  	listFile := filepath.Join(dir, "list")
   707  
   708  	// Lock listfile when writing to it to try to avoid corruption to the file.
   709  	// Under rare circumstances, for instance, if the system loses power in the
   710  	// middle of a write it is possible for corrupt data to be written. This is
   711  	// not a problem for the go command itself, but may be an issue if the
   712  	// cache is being served by a GOPROXY HTTP server. This will be corrected
   713  	// the next time a new version of the module is fetched and the file is rewritten.
   714  	// TODO(matloob): golang.org/issue/43313 covers adding a go mod verify
   715  	// command that removes module versions that fail checksums. It should also
   716  	// remove list files that are detected to be corrupt.
   717  	f, err := lockedfile.Edit(listFile)
   718  	if err != nil {
   719  		return err
   720  	}
   721  	defer func() {
   722  		if cerr := f.Close(); cerr != nil && err == nil {
   723  			err = cerr
   724  		}
   725  	}()
   726  	infos, err := os.ReadDir(dir)
   727  	if err != nil {
   728  		return err
   729  	}
   730  	var list []string
   731  	for _, info := range infos {
   732  		// We look for *.mod files on the theory that if we can't supply
   733  		// the .mod file then there's no point in listing that version,
   734  		// since it's unusable. (We can have *.info without *.mod.)
   735  		// We don't require *.zip files on the theory that for code only
   736  		// involved in module graph construction, many *.zip files
   737  		// will never be requested.
   738  		name := info.Name()
   739  		if v, found := strings.CutSuffix(name, ".mod"); found {
   740  			if v != "" && module.CanonicalVersion(v) == v {
   741  				list = append(list, v)
   742  			}
   743  		}
   744  	}
   745  	semver.Sort(list)
   746  
   747  	var buf bytes.Buffer
   748  	for _, v := range list {
   749  		buf.WriteString(v)
   750  		buf.WriteString("\n")
   751  	}
   752  	if fi, err := f.Stat(); err == nil && int(fi.Size()) == buf.Len() {
   753  		old := make([]byte, buf.Len()+1)
   754  		if n, err := f.ReadAt(old, 0); err == io.EOF && n == buf.Len() && bytes.Equal(buf.Bytes(), old) {
   755  			return nil // No edit needed.
   756  		}
   757  	}
   758  	// Remove existing contents, so that when we truncate to the actual size it will zero-fill,
   759  	// and we will be able to detect (some) incomplete writes as files containing trailing NUL bytes.
   760  	if err := f.Truncate(0); err != nil {
   761  		return err
   762  	}
   763  	// Reserve the final size and zero-fill.
   764  	if err := f.Truncate(int64(buf.Len())); err != nil {
   765  		return err
   766  	}
   767  	// Write the actual contents. If this fails partway through,
   768  	// the remainder of the file should remain as zeroes.
   769  	if _, err := f.Write(buf.Bytes()); err != nil {
   770  		f.Truncate(0)
   771  		return err
   772  	}
   773  
   774  	return nil
   775  }
   776  
   777  var (
   778  	statCacheOnce sync.Once
   779  	statCacheErr  error
   780  )
   781  
   782  // checkCacheDir checks if the directory specified by GOMODCACHE exists. An
   783  // error is returned if it does not.
   784  func checkCacheDir(ctx context.Context) error {
   785  	if cfg.GOMODCACHE == "" {
   786  		// modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE
   787  		// is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen.
   788  		return fmt.Errorf("module cache not found: neither GOMODCACHE nor GOPATH is set")
   789  	}
   790  	if !filepath.IsAbs(cfg.GOMODCACHE) {
   791  		return fmt.Errorf("GOMODCACHE entry is relative; must be absolute path: %q.\n", cfg.GOMODCACHE)
   792  	}
   793  
   794  	// os.Stat is slow on Windows, so we only call it once to prevent unnecessary
   795  	// I/O every time this function is called.
   796  	statCacheOnce.Do(func() {
   797  		fi, err := os.Stat(cfg.GOMODCACHE)
   798  		if err != nil {
   799  			if !os.IsNotExist(err) {
   800  				statCacheErr = fmt.Errorf("could not create module cache: %w", err)
   801  				return
   802  			}
   803  			if err := os.MkdirAll(cfg.GOMODCACHE, 0777); err != nil {
   804  				statCacheErr = fmt.Errorf("could not create module cache: %w", err)
   805  				return
   806  			}
   807  			return
   808  		}
   809  		if !fi.IsDir() {
   810  			statCacheErr = fmt.Errorf("could not create module cache: %q is not a directory", cfg.GOMODCACHE)
   811  			return
   812  		}
   813  	})
   814  	return statCacheErr
   815  }
   816
View as plain text