gofmt.go

Documentation: cmd/gofmt

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"flag"
    11  	"fmt"
    12  	"go/ast"
    13  	"go/parser"
    14  	"go/printer"
    15  	"go/scanner"
    16  	"go/token"
    17  	"internal/diff"
    18  	"io"
    19  	"io/fs"
    20  	"math/rand"
    21  	"os"
    22  	"path/filepath"
    23  	"runtime"
    24  	"runtime/pprof"
    25  	"strconv"
    26  	"strings"
    27  
    28  	"golang.org/x/sync/semaphore"
    29  )
    30  
    31  var (
    32  	// main operation modes
    33  	list        = flag.Bool("l", false, "list files whose formatting differs from gofmt's")
    34  	write       = flag.Bool("w", false, "write result to (source) file instead of stdout")
    35  	rewriteRule = flag.String("r", "", "rewrite rule (e.g., 'a[b:len(a)] -> a[b:]')")
    36  	simplifyAST = flag.Bool("s", false, "simplify code")
    37  	doDiff      = flag.Bool("d", false, "display diffs instead of rewriting files")
    38  	allErrors   = flag.Bool("e", false, "report all errors (not just the first 10 on different lines)")
    39  
    40  	// debugging
    41  	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to this file")
    42  )
    43  
    44  // Keep these in sync with go/format/format.go.
    45  const (
    46  	tabWidth    = 8
    47  	printerMode = printer.UseSpaces | printer.TabIndent | printerNormalizeNumbers
    48  
    49  	// printerNormalizeNumbers means to canonicalize number literal prefixes
    50  	// and exponents while printing. See https://golang.org/doc/go1.13#gofmt.
    51  	//
    52  	// This value is defined in go/printer specifically for go/format and cmd/gofmt.
    53  	printerNormalizeNumbers = 1 << 30
    54  )
    55  
    56  // fdSem guards the number of concurrently-open file descriptors.
    57  //
    58  // For now, this is arbitrarily set to 200, based on the observation that many
    59  // platforms default to a kernel limit of 256. Ideally, perhaps we should derive
    60  // it from rlimit on platforms that support that system call.
    61  //
    62  // File descriptors opened from outside of this package are not tracked,
    63  // so this limit may be approximate.
    64  var fdSem = make(chan bool, 200)
    65  
    66  var (
    67  	rewrite    func(*token.FileSet, *ast.File) *ast.File
    68  	parserMode parser.Mode
    69  )
    70  
    71  func usage() {
    72  	fmt.Fprintf(os.Stderr, "usage: gofmt [flags] [path ...]\n")
    73  	flag.PrintDefaults()
    74  }
    75  
    76  func initParserMode() {
    77  	parserMode = parser.ParseComments
    78  	if *allErrors {
    79  		parserMode |= parser.AllErrors
    80  	}
    81  	// It's only -r that makes use of go/ast's object resolution,
    82  	// so avoid the unnecessary work if the flag isn't used.
    83  	if *rewriteRule == "" {
    84  		parserMode |= parser.SkipObjectResolution
    85  	}
    86  }
    87  
    88  func isGoFile(f fs.DirEntry) bool {
    89  	// ignore non-Go files
    90  	name := f.Name()
    91  	return !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go") && !f.IsDir()
    92  }
    93  
    94  // A sequencer performs concurrent tasks that may write output, but emits that
    95  // output in a deterministic order.
    96  type sequencer struct {
    97  	maxWeight int64
    98  	sem       *semaphore.Weighted   // weighted by input bytes (an approximate proxy for memory overhead)
    99  	prev      <-chan *reporterState // 1-buffered
   100  }
   101  
   102  // newSequencer returns a sequencer that allows concurrent tasks up to maxWeight
   103  // and writes tasks' output to out and err.
   104  func newSequencer(maxWeight int64, out, err io.Writer) *sequencer {
   105  	sem := semaphore.NewWeighted(maxWeight)
   106  	prev := make(chan *reporterState, 1)
   107  	prev <- &reporterState{out: out, err: err}
   108  	return &sequencer{
   109  		maxWeight: maxWeight,
   110  		sem:       sem,
   111  		prev:      prev,
   112  	}
   113  }
   114  
   115  // exclusive is a weight that can be passed to a sequencer to cause
   116  // a task to be executed without any other concurrent tasks.
   117  const exclusive = -1
   118  
   119  // Add blocks until the sequencer has enough weight to spare, then adds f as a
   120  // task to be executed concurrently.
   121  //
   122  // If the weight is either negative or larger than the sequencer's maximum
   123  // weight, Add blocks until all other tasks have completed, then the task
   124  // executes exclusively (blocking all other calls to Add until it completes).
   125  //
   126  // f may run concurrently in a goroutine, but its output to the passed-in
   127  // reporter will be sequential relative to the other tasks in the sequencer.
   128  //
   129  // If f invokes a method on the reporter, execution of that method may block
   130  // until the previous task has finished. (To maximize concurrency, f should
   131  // avoid invoking the reporter until it has finished any parallelizable work.)
   132  //
   133  // If f returns a non-nil error, that error will be reported after f's output
   134  // (if any) and will cause a nonzero final exit code.
   135  func (s *sequencer) Add(weight int64, f func(*reporter) error) {
   136  	if weight < 0 || weight > s.maxWeight {
   137  		weight = s.maxWeight
   138  	}
   139  	if err := s.sem.Acquire(context.TODO(), weight); err != nil {
   140  		// Change the task from "execute f" to "report err".
   141  		weight = 0
   142  		f = func(*reporter) error { return err }
   143  	}
   144  
   145  	r := &reporter{prev: s.prev}
   146  	next := make(chan *reporterState, 1)
   147  	s.prev = next
   148  
   149  	// Start f in parallel: it can run until it invokes a method on r, at which
   150  	// point it will block until the previous task releases the output state.
   151  	go func() {
   152  		if err := f(r); err != nil {
   153  			r.Report(err)
   154  		}
   155  		next <- r.getState() // Release the next task.
   156  		s.sem.Release(weight)
   157  	}()
   158  }
   159  
   160  // AddReport prints an error to s after the output of any previously-added
   161  // tasks, causing the final exit code to be nonzero.
   162  func (s *sequencer) AddReport(err error) {
   163  	s.Add(0, func(*reporter) error { return err })
   164  }
   165  
   166  // GetExitCode waits for all previously-added tasks to complete, then returns an
   167  // exit code for the sequence suitable for passing to os.Exit.
   168  func (s *sequencer) GetExitCode() int {
   169  	c := make(chan int, 1)
   170  	s.Add(0, func(r *reporter) error {
   171  		c <- r.ExitCode()
   172  		return nil
   173  	})
   174  	return <-c
   175  }
   176  
   177  // A reporter reports output, warnings, and errors.
   178  type reporter struct {
   179  	prev  <-chan *reporterState
   180  	state *reporterState
   181  }
   182  
   183  // reporterState carries the state of a reporter instance.
   184  //
   185  // Only one reporter at a time may have access to a reporterState.
   186  type reporterState struct {
   187  	out, err io.Writer
   188  	exitCode int
   189  }
   190  
   191  // getState blocks until any prior reporters are finished with the reporter
   192  // state, then returns the state for manipulation.
   193  func (r *reporter) getState() *reporterState {
   194  	if r.state == nil {
   195  		r.state = <-r.prev
   196  	}
   197  	return r.state
   198  }
   199  
   200  // Warnf emits a warning message to the reporter's error stream,
   201  // without changing its exit code.
   202  func (r *reporter) Warnf(format string, args ...any) {
   203  	fmt.Fprintf(r.getState().err, format, args...)
   204  }
   205  
   206  // Write emits a slice to the reporter's output stream.
   207  //
   208  // Any error is returned to the caller, and does not otherwise affect the
   209  // reporter's exit code.
   210  func (r *reporter) Write(p []byte) (int, error) {
   211  	return r.getState().out.Write(p)
   212  }
   213  
   214  // Report emits a non-nil error to the reporter's error stream,
   215  // changing its exit code to a nonzero value.
   216  func (r *reporter) Report(err error) {
   217  	if err == nil {
   218  		panic("Report with nil error")
   219  	}
   220  	st := r.getState()
   221  	scanner.PrintError(st.err, err)
   222  	st.exitCode = 2
   223  }
   224  
   225  func (r *reporter) ExitCode() int {
   226  	return r.getState().exitCode
   227  }
   228  
   229  // If info == nil, we are formatting stdin instead of a file.
   230  // If in == nil, the source is the contents of the file with the given filename.
   231  func processFile(filename string, info fs.FileInfo, in io.Reader, r *reporter) error {
   232  	src, err := readFile(filename, info, in)
   233  	if err != nil {
   234  		return err
   235  	}
   236  
   237  	fileSet := token.NewFileSet()
   238  	// If we are formatting stdin, we accept a program fragment in lieu of a
   239  	// complete source file.
   240  	fragmentOk := info == nil
   241  	file, sourceAdj, indentAdj, err := parse(fileSet, filename, src, fragmentOk)
   242  	if err != nil {
   243  		return err
   244  	}
   245  
   246  	if rewrite != nil {
   247  		if sourceAdj == nil {
   248  			file = rewrite(fileSet, file)
   249  		} else {
   250  			r.Warnf("warning: rewrite ignored for incomplete programs\n")
   251  		}
   252  	}
   253  
   254  	ast.SortImports(fileSet, file)
   255  
   256  	if *simplifyAST {
   257  		simplify(file)
   258  	}
   259  
   260  	res, err := format(fileSet, file, sourceAdj, indentAdj, src, printer.Config{Mode: printerMode, Tabwidth: tabWidth})
   261  	if err != nil {
   262  		return err
   263  	}
   264  
   265  	if !bytes.Equal(src, res) {
   266  		// formatting has changed
   267  		if *list {
   268  			fmt.Fprintln(r, filename)
   269  		}
   270  		if *write {
   271  			if info == nil {
   272  				panic("-w should not have been allowed with stdin")
   273  			}
   274  
   275  			perm := info.Mode().Perm()
   276  			if err := writeFile(filename, src, res, perm, info.Size()); err != nil {
   277  				return err
   278  			}
   279  		}
   280  		if *doDiff {
   281  			newName := filepath.ToSlash(filename)
   282  			oldName := newName + ".orig"
   283  			r.Write(diff.Diff(oldName, src, newName, res))
   284  		}
   285  	}
   286  
   287  	if !*list && !*write && !*doDiff {
   288  		_, err = r.Write(res)
   289  	}
   290  
   291  	return err
   292  }
   293  
   294  // readFile reads the contents of filename, described by info.
   295  // If in is non-nil, readFile reads directly from it.
   296  // Otherwise, readFile opens and reads the file itself,
   297  // with the number of concurrently-open files limited by fdSem.
   298  func readFile(filename string, info fs.FileInfo, in io.Reader) ([]byte, error) {
   299  	if in == nil {
   300  		fdSem <- true
   301  		var err error
   302  		f, err := os.Open(filename)
   303  		if err != nil {
   304  			return nil, err
   305  		}
   306  		in = f
   307  		defer func() {
   308  			f.Close()
   309  			<-fdSem
   310  		}()
   311  	}
   312  
   313  	// Compute the file's size and read its contents with minimal allocations.
   314  	//
   315  	// If we have the FileInfo from filepath.WalkDir, use it to make
   316  	// a buffer of the right size and avoid ReadAll's reallocations.
   317  	//
   318  	// If the size is unknown (or bogus, or overflows an int), fall back to
   319  	// a size-independent ReadAll.
   320  	size := -1
   321  	if info != nil && info.Mode().IsRegular() && int64(int(info.Size())) == info.Size() {
   322  		size = int(info.Size())
   323  	}
   324  	if size+1 <= 0 {
   325  		// The file is not known to be regular, so we don't have a reliable size for it.
   326  		var err error
   327  		src, err := io.ReadAll(in)
   328  		if err != nil {
   329  			return nil, err
   330  		}
   331  		return src, nil
   332  	}
   333  
   334  	// We try to read size+1 bytes so that we can detect modifications: if we
   335  	// read more than size bytes, then the file was modified concurrently.
   336  	// (If that happens, we could, say, append to src to finish the read, or
   337  	// proceed with a truncated buffer — but the fact that it changed at all
   338  	// indicates a possible race with someone editing the file, so we prefer to
   339  	// stop to avoid corrupting it.)
   340  	src := make([]byte, size+1)
   341  	n, err := io.ReadFull(in, src)
   342  	switch err {
   343  	case nil, io.EOF, io.ErrUnexpectedEOF:
   344  		// io.ReadFull returns io.EOF (for an empty file) or io.ErrUnexpectedEOF
   345  		// (for a non-empty file) if the file was changed unexpectedly. Continue
   346  		// with comparing file sizes in those cases.
   347  	default:
   348  		return nil, err
   349  	}
   350  	if n < size {
   351  		return nil, fmt.Errorf("error: size of %s changed during reading (from %d to %d bytes)", filename, size, n)
   352  	} else if n > size {
   353  		return nil, fmt.Errorf("error: size of %s changed during reading (from %d to >=%d bytes)", filename, size, len(src))
   354  	}
   355  	return src[:n], nil
   356  }
   357  
   358  func main() {
   359  	// Arbitrarily limit in-flight work to 2MiB times the number of threads.
   360  	//
   361  	// The actual overhead for the parse tree and output will depend on the
   362  	// specifics of the file, but this at least keeps the footprint of the process
   363  	// roughly proportional to GOMAXPROCS.
   364  	maxWeight := (2 << 20) * int64(runtime.GOMAXPROCS(0))
   365  	s := newSequencer(maxWeight, os.Stdout, os.Stderr)
   366  
   367  	// call gofmtMain in a separate function
   368  	// so that it can use defer and have them
   369  	// run before the exit.
   370  	gofmtMain(s)
   371  	os.Exit(s.GetExitCode())
   372  }
   373  
   374  func gofmtMain(s *sequencer) {
   375  	flag.Usage = usage
   376  	flag.Parse()
   377  
   378  	if *cpuprofile != "" {
   379  		fdSem <- true
   380  		f, err := os.Create(*cpuprofile)
   381  		if err != nil {
   382  			s.AddReport(fmt.Errorf("creating cpu profile: %s", err))
   383  			return
   384  		}
   385  		defer func() {
   386  			f.Close()
   387  			<-fdSem
   388  		}()
   389  		pprof.StartCPUProfile(f)
   390  		defer pprof.StopCPUProfile()
   391  	}
   392  
   393  	initParserMode()
   394  	initRewrite()
   395  
   396  	args := flag.Args()
   397  	if len(args) == 0 {
   398  		if *write {
   399  			s.AddReport(fmt.Errorf("error: cannot use -w with standard input"))
   400  			return
   401  		}
   402  		s.Add(0, func(r *reporter) error {
   403  			return processFile("<standard input>", nil, os.Stdin, r)
   404  		})
   405  		return
   406  	}
   407  
   408  	for _, arg := range args {
   409  		switch info, err := os.Stat(arg); {
   410  		case err != nil:
   411  			s.AddReport(err)
   412  		case !info.IsDir():
   413  			// Non-directory arguments are always formatted.
   414  			arg := arg
   415  			s.Add(fileWeight(arg, info), func(r *reporter) error {
   416  				return processFile(arg, info, nil, r)
   417  			})
   418  		default:
   419  			// Directories are walked, ignoring non-Go files.
   420  			err := filepath.WalkDir(arg, func(path string, f fs.DirEntry, err error) error {
   421  				if err != nil || !isGoFile(f) {
   422  					return err
   423  				}
   424  				info, err := f.Info()
   425  				if err != nil {
   426  					s.AddReport(err)
   427  					return nil
   428  				}
   429  				s.Add(fileWeight(path, info), func(r *reporter) error {
   430  					return processFile(path, info, nil, r)
   431  				})
   432  				return nil
   433  			})
   434  			if err != nil {
   435  				s.AddReport(err)
   436  			}
   437  		}
   438  	}
   439  }
   440  
   441  func fileWeight(path string, info fs.FileInfo) int64 {
   442  	if info == nil {
   443  		return exclusive
   444  	}
   445  	if info.Mode().Type() == fs.ModeSymlink {
   446  		var err error
   447  		info, err = os.Stat(path)
   448  		if err != nil {
   449  			return exclusive
   450  		}
   451  	}
   452  	if !info.Mode().IsRegular() {
   453  		// For non-regular files, FileInfo.Size is system-dependent and thus not a
   454  		// reliable indicator of weight.
   455  		return exclusive
   456  	}
   457  	return info.Size()
   458  }
   459  
   460  // writeFile updates a file with the new formatted data.
   461  func writeFile(filename string, orig, formatted []byte, perm fs.FileMode, size int64) error {
   462  	// Make a temporary backup file before rewriting the original file.
   463  	bakname, err := backupFile(filename, orig, perm)
   464  	if err != nil {
   465  		return err
   466  	}
   467  
   468  	fdSem <- true
   469  	defer func() { <-fdSem }()
   470  
   471  	fout, err := os.OpenFile(filename, os.O_WRONLY, perm)
   472  	if err != nil {
   473  		// We couldn't even open the file, so it should
   474  		// not have changed.
   475  		os.Remove(bakname)
   476  		return err
   477  	}
   478  	defer fout.Close() // for error paths
   479  
   480  	restoreFail := func(err error) {
   481  		fmt.Fprintf(os.Stderr, "gofmt: %s: error restoring file to original: %v; backup in %s\n", filename, err, bakname)
   482  	}
   483  
   484  	n, err := fout.Write(formatted)
   485  	if err == nil && int64(n) < size {
   486  		err = fout.Truncate(int64(n))
   487  	}
   488  
   489  	if err != nil {
   490  		// Rewriting the file failed.
   491  
   492  		if n == 0 {
   493  			// Original file unchanged.
   494  			os.Remove(bakname)
   495  			return err
   496  		}
   497  
   498  		// Try to restore the original contents.
   499  
   500  		no, erro := fout.WriteAt(orig, 0)
   501  		if erro != nil {
   502  			// That failed too.
   503  			restoreFail(erro)
   504  			return err
   505  		}
   506  
   507  		if no < n {
   508  			// Original file is shorter. Truncate.
   509  			if erro = fout.Truncate(int64(no)); erro != nil {
   510  				restoreFail(erro)
   511  				return err
   512  			}
   513  		}
   514  
   515  		if erro := fout.Close(); erro != nil {
   516  			restoreFail(erro)
   517  			return err
   518  		}
   519  
   520  		// Original contents restored.
   521  		os.Remove(bakname)
   522  		return err
   523  	}
   524  
   525  	if err := fout.Close(); err != nil {
   526  		restoreFail(err)
   527  		return err
   528  	}
   529  
   530  	// File updated.
   531  	os.Remove(bakname)
   532  	return nil
   533  }
   534  
   535  // backupFile writes data to a new file named filename<number> with permissions perm,
   536  // with <number> randomly chosen such that the file name is unique. backupFile returns
   537  // the chosen file name.
   538  func backupFile(filename string, data []byte, perm fs.FileMode) (string, error) {
   539  	fdSem <- true
   540  	defer func() { <-fdSem }()
   541  
   542  	nextRandom := func() string {
   543  		return strconv.Itoa(rand.Int())
   544  	}
   545  
   546  	dir, base := filepath.Split(filename)
   547  	var (
   548  		bakname string
   549  		f       *os.File
   550  	)
   551  	for {
   552  		bakname = filepath.Join(dir, base+"."+nextRandom())
   553  		var err error
   554  		f, err = os.OpenFile(bakname, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm)
   555  		if err == nil {
   556  			break
   557  		}
   558  		if err != nil && !os.IsExist(err) {
   559  			return "", err
   560  		}
   561  	}
   562  
   563  	// write data to backup file
   564  	_, err := f.Write(data)
   565  	if err1 := f.Close(); err == nil {
   566  		err = err1
   567  	}
   568  
   569  	return bakname, err
   570  }
   571
View as plain text