gofmt.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"flag"
    11  	"fmt"
    12  	"go/ast"
    13  	"go/parser"
    14  	"go/printer"
    15  	"go/scanner"
    16  	"go/token"
    17  	"internal/diff"
    18  	"io"
    19  	"io/fs"
    20  	"math/rand"
    21  	"os"
    22  	"path/filepath"
    23  	"runtime"
    24  	"runtime/pprof"
    25  	"strconv"
    26  	"strings"
    27  
    28  	"cmd/internal/telemetry/counter"
    29  
    30  	"golang.org/x/sync/semaphore"
    31  )
    32  
    33  var (
    34  	// main operation modes
    35  	list        = flag.Bool("l", false, "list files whose formatting differs from gofmt's")
    36  	write       = flag.Bool("w", false, "write result to (source) file instead of stdout")
    37  	rewriteRule = flag.String("r", "", "rewrite rule (e.g., 'a[b:len(a)] -> a[b:]')")
    38  	simplifyAST = flag.Bool("s", false, "simplify code")
    39  	doDiff      = flag.Bool("d", false, "display diffs instead of rewriting files")
    40  	allErrors   = flag.Bool("e", false, "report all errors (not just the first 10 on different lines)")
    41  
    42  	// debugging
    43  	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to this file")
    44  
    45  	// errors
    46  	errFormattingDiffers = fmt.Errorf("formatting differs from gofmt's")
    47  )
    48  
    49  // Keep these in sync with go/format/format.go.
    50  const (
    51  	tabWidth    = 8
    52  	printerMode = printer.UseSpaces | printer.TabIndent | printerNormalizeNumbers
    53  
    54  	// printerNormalizeNumbers means to canonicalize number literal prefixes
    55  	// and exponents while printing. See https://golang.org/doc/go1.13#gofmt.
    56  	//
    57  	// This value is defined in go/printer specifically for go/format and cmd/gofmt.
    58  	printerNormalizeNumbers = 1 << 30
    59  )
    60  
    61  // fdSem guards the number of concurrently-open file descriptors.
    62  //
    63  // For now, this is arbitrarily set to 200, based on the observation that many
    64  // platforms default to a kernel limit of 256. Ideally, perhaps we should derive
    65  // it from rlimit on platforms that support that system call.
    66  //
    67  // File descriptors opened from outside of this package are not tracked,
    68  // so this limit may be approximate.
    69  var fdSem = make(chan bool, 200)
    70  
    71  var (
    72  	rewrite    func(*token.FileSet, *ast.File) *ast.File
    73  	parserMode parser.Mode
    74  )
    75  
    76  func usage() {
    77  	fmt.Fprintf(os.Stderr, "usage: gofmt [flags] [path ...]\n")
    78  	flag.PrintDefaults()
    79  }
    80  
    81  func initParserMode() {
    82  	parserMode = parser.ParseComments
    83  	if *allErrors {
    84  		parserMode |= parser.AllErrors
    85  	}
    86  	// It's only -r that makes use of go/ast's object resolution,
    87  	// so avoid the unnecessary work if the flag isn't used.
    88  	if *rewriteRule == "" {
    89  		parserMode |= parser.SkipObjectResolution
    90  	}
    91  }
    92  
    93  func isGoFilename(name string) bool {
    94  	return !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go")
    95  }
    96  
    97  // A sequencer performs concurrent tasks that may write output, but emits that
    98  // output in a deterministic order.
    99  type sequencer struct {
   100  	maxWeight int64
   101  	sem       *semaphore.Weighted   // weighted by input bytes (an approximate proxy for memory overhead)
   102  	prev      <-chan *reporterState // 1-buffered
   103  }
   104  
   105  // newSequencer returns a sequencer that allows concurrent tasks up to maxWeight
   106  // and writes tasks' output to out and err.
   107  func newSequencer(maxWeight int64, out, err io.Writer) *sequencer {
   108  	sem := semaphore.NewWeighted(maxWeight)
   109  	prev := make(chan *reporterState, 1)
   110  	prev <- &reporterState{out: out, err: err}
   111  	return &sequencer{
   112  		maxWeight: maxWeight,
   113  		sem:       sem,
   114  		prev:      prev,
   115  	}
   116  }
   117  
   118  // exclusive is a weight that can be passed to a sequencer to cause
   119  // a task to be executed without any other concurrent tasks.
   120  const exclusive = -1
   121  
   122  // Add blocks until the sequencer has enough weight to spare, then adds f as a
   123  // task to be executed concurrently.
   124  //
   125  // If the weight is either negative or larger than the sequencer's maximum
   126  // weight, Add blocks until all other tasks have completed, then the task
   127  // executes exclusively (blocking all other calls to Add until it completes).
   128  //
   129  // f may run concurrently in a goroutine, but its output to the passed-in
   130  // reporter will be sequential relative to the other tasks in the sequencer.
   131  //
   132  // If f invokes a method on the reporter, execution of that method may block
   133  // until the previous task has finished. (To maximize concurrency, f should
   134  // avoid invoking the reporter until it has finished any parallelizable work.)
   135  //
   136  // If f returns a non-nil error, that error will be reported after f's output
   137  // (if any) and will cause a nonzero final exit code.
   138  func (s *sequencer) Add(weight int64, f func(*reporter) error) {
   139  	if weight < 0 || weight > s.maxWeight {
   140  		weight = s.maxWeight
   141  	}
   142  	if err := s.sem.Acquire(context.TODO(), weight); err != nil {
   143  		// Change the task from "execute f" to "report err".
   144  		weight = 0
   145  		f = func(*reporter) error { return err }
   146  	}
   147  
   148  	r := &reporter{prev: s.prev}
   149  	next := make(chan *reporterState, 1)
   150  	s.prev = next
   151  
   152  	// Start f in parallel: it can run until it invokes a method on r, at which
   153  	// point it will block until the previous task releases the output state.
   154  	go func() {
   155  		if err := f(r); err != nil {
   156  			r.Report(err)
   157  		}
   158  		next <- r.getState() // Release the next task.
   159  		s.sem.Release(weight)
   160  	}()
   161  }
   162  
   163  // AddReport prints an error to s after the output of any previously-added
   164  // tasks, causing the final exit code to be nonzero.
   165  func (s *sequencer) AddReport(err error) {
   166  	s.Add(0, func(*reporter) error { return err })
   167  }
   168  
   169  // GetExitCode waits for all previously-added tasks to complete, then returns an
   170  // exit code for the sequence suitable for passing to os.Exit.
   171  func (s *sequencer) GetExitCode() int {
   172  	c := make(chan int, 1)
   173  	s.Add(0, func(r *reporter) error {
   174  		c <- r.ExitCode()
   175  		return nil
   176  	})
   177  	return <-c
   178  }
   179  
   180  // A reporter reports output, warnings, and errors.
   181  type reporter struct {
   182  	prev  <-chan *reporterState
   183  	state *reporterState
   184  }
   185  
   186  // reporterState carries the state of a reporter instance.
   187  //
   188  // Only one reporter at a time may have access to a reporterState.
   189  type reporterState struct {
   190  	out, err io.Writer
   191  	exitCode int
   192  }
   193  
   194  // getState blocks until any prior reporters are finished with the reporter
   195  // state, then returns the state for manipulation.
   196  func (r *reporter) getState() *reporterState {
   197  	if r.state == nil {
   198  		r.state = <-r.prev
   199  	}
   200  	return r.state
   201  }
   202  
   203  // Warnf emits a warning message to the reporter's error stream,
   204  // without changing its exit code.
   205  func (r *reporter) Warnf(format string, args ...any) {
   206  	fmt.Fprintf(r.getState().err, format, args...)
   207  }
   208  
   209  // Write emits a slice to the reporter's output stream.
   210  //
   211  // Any error is returned to the caller, and does not otherwise affect the
   212  // reporter's exit code.
   213  func (r *reporter) Write(p []byte) (int, error) {
   214  	return r.getState().out.Write(p)
   215  }
   216  
   217  // Report emits a non-nil error to the reporter's error stream,
   218  // changing its exit code to a nonzero value.
   219  func (r *reporter) Report(err error) {
   220  	if err == nil {
   221  		panic("Report with nil error")
   222  	}
   223  	st := r.getState()
   224  	if err == errFormattingDiffers {
   225  		st.exitCode = 1
   226  	} else {
   227  		scanner.PrintError(st.err, err)
   228  		st.exitCode = 2
   229  	}
   230  }
   231  
   232  func (r *reporter) ExitCode() int {
   233  	return r.getState().exitCode
   234  }
   235  
   236  // If info == nil, we are formatting stdin instead of a file.
   237  // If in == nil, the source is the contents of the file with the given filename.
   238  func processFile(filename string, info fs.FileInfo, in io.Reader, r *reporter) error {
   239  	src, err := readFile(filename, info, in)
   240  	if err != nil {
   241  		return err
   242  	}
   243  
   244  	fileSet := token.NewFileSet()
   245  	// If we are formatting stdin, we accept a program fragment in lieu of a
   246  	// complete source file.
   247  	fragmentOk := info == nil
   248  	file, sourceAdj, indentAdj, err := parse(fileSet, filename, src, fragmentOk)
   249  	if err != nil {
   250  		return err
   251  	}
   252  
   253  	if rewrite != nil {
   254  		if sourceAdj == nil {
   255  			file = rewrite(fileSet, file)
   256  		} else {
   257  			r.Warnf("warning: rewrite ignored for incomplete programs\n")
   258  		}
   259  	}
   260  
   261  	ast.SortImports(fileSet, file)
   262  
   263  	if *simplifyAST {
   264  		simplify(file)
   265  	}
   266  
   267  	res, err := format(fileSet, file, sourceAdj, indentAdj, src, printer.Config{Mode: printerMode, Tabwidth: tabWidth})
   268  	if err != nil {
   269  		return err
   270  	}
   271  
   272  	if !bytes.Equal(src, res) {
   273  		// formatting has changed
   274  		if *list {
   275  			fmt.Fprintln(r, filename)
   276  		}
   277  		if *write {
   278  			if info == nil {
   279  				panic("-w should not have been allowed with stdin")
   280  			}
   281  
   282  			perm := info.Mode().Perm()
   283  			if err := writeFile(filename, src, res, perm, info.Size()); err != nil {
   284  				return err
   285  			}
   286  		}
   287  		if *doDiff {
   288  			newName := filepath.ToSlash(filename)
   289  			oldName := newName + ".orig"
   290  			r.Write(diff.Diff(oldName, src, newName, res))
   291  			return errFormattingDiffers
   292  		}
   293  	}
   294  
   295  	if !*list && !*write && !*doDiff {
   296  		_, err = r.Write(res)
   297  	}
   298  
   299  	return err
   300  }
   301  
   302  // readFile reads the contents of filename, described by info.
   303  // If in is non-nil, readFile reads directly from it.
   304  // Otherwise, readFile opens and reads the file itself,
   305  // with the number of concurrently-open files limited by fdSem.
   306  func readFile(filename string, info fs.FileInfo, in io.Reader) ([]byte, error) {
   307  	if in == nil {
   308  		fdSem <- true
   309  		var err error
   310  		f, err := os.Open(filename)
   311  		if err != nil {
   312  			return nil, err
   313  		}
   314  		in = f
   315  		defer func() {
   316  			f.Close()
   317  			<-fdSem
   318  		}()
   319  	}
   320  
   321  	// Compute the file's size and read its contents with minimal allocations.
   322  	//
   323  	// If we have the FileInfo from filepath.WalkDir, use it to make
   324  	// a buffer of the right size and avoid ReadAll's reallocations.
   325  	//
   326  	// If the size is unknown (or bogus, or overflows an int), fall back to
   327  	// a size-independent ReadAll.
   328  	size := -1
   329  	if info != nil && info.Mode().IsRegular() && int64(int(info.Size())) == info.Size() {
   330  		size = int(info.Size())
   331  	}
   332  	if size+1 <= 0 {
   333  		// The file is not known to be regular, so we don't have a reliable size for it.
   334  		var err error
   335  		src, err := io.ReadAll(in)
   336  		if err != nil {
   337  			return nil, err
   338  		}
   339  		return src, nil
   340  	}
   341  
   342  	// We try to read size+1 bytes so that we can detect modifications: if we
   343  	// read more than size bytes, then the file was modified concurrently.
   344  	// (If that happens, we could, say, append to src to finish the read, or
   345  	// proceed with a truncated buffer — but the fact that it changed at all
   346  	// indicates a possible race with someone editing the file, so we prefer to
   347  	// stop to avoid corrupting it.)
   348  	src := make([]byte, size+1)
   349  	n, err := io.ReadFull(in, src)
   350  	switch err {
   351  	case nil, io.EOF, io.ErrUnexpectedEOF:
   352  		// io.ReadFull returns io.EOF (for an empty file) or io.ErrUnexpectedEOF
   353  		// (for a non-empty file) if the file was changed unexpectedly. Continue
   354  		// with comparing file sizes in those cases.
   355  	default:
   356  		return nil, err
   357  	}
   358  	if n < size {
   359  		return nil, fmt.Errorf("error: size of %s changed during reading (from %d to %d bytes)", filename, size, n)
   360  	} else if n > size {
   361  		return nil, fmt.Errorf("error: size of %s changed during reading (from %d to >=%d bytes)", filename, size, len(src))
   362  	}
   363  	return src[:n], nil
   364  }
   365  
   366  func main() {
   367  	// Arbitrarily limit in-flight work to 2MiB times the number of threads.
   368  	//
   369  	// The actual overhead for the parse tree and output will depend on the
   370  	// specifics of the file, but this at least keeps the footprint of the process
   371  	// roughly proportional to GOMAXPROCS.
   372  	maxWeight := (2 << 20) * int64(runtime.GOMAXPROCS(0))
   373  	s := newSequencer(maxWeight, os.Stdout, os.Stderr)
   374  
   375  	// call gofmtMain in a separate function
   376  	// so that it can use defer and have them
   377  	// run before the exit.
   378  	gofmtMain(s)
   379  	os.Exit(s.GetExitCode())
   380  }
   381  
   382  func gofmtMain(s *sequencer) {
   383  	counter.Open()
   384  	flag.Usage = usage
   385  	flag.Parse()
   386  	counter.Inc("gofmt/invocations")
   387  	counter.CountFlags("gofmt/flag:", *flag.CommandLine)
   388  
   389  	if *cpuprofile != "" {
   390  		fdSem <- true
   391  		f, err := os.Create(*cpuprofile)
   392  		if err != nil {
   393  			s.AddReport(fmt.Errorf("creating cpu profile: %s", err))
   394  			return
   395  		}
   396  		defer func() {
   397  			f.Close()
   398  			<-fdSem
   399  		}()
   400  		pprof.StartCPUProfile(f)
   401  		defer pprof.StopCPUProfile()
   402  	}
   403  
   404  	initParserMode()
   405  	initRewrite()
   406  
   407  	args := flag.Args()
   408  	if len(args) == 0 {
   409  		if *write {
   410  			s.AddReport(fmt.Errorf("error: cannot use -w with standard input"))
   411  			return
   412  		}
   413  		s.Add(0, func(r *reporter) error {
   414  			return processFile("<standard input>", nil, os.Stdin, r)
   415  		})
   416  		return
   417  	}
   418  
   419  	for _, arg := range args {
   420  		// Walk each given argument as a directory tree.
   421  		// If the argument is not a directory, it's always formatted as a Go file.
   422  		// If the argument is a directory, we walk it, ignoring non-Go files.
   423  		if err := filepath.WalkDir(arg, func(path string, d fs.DirEntry, err error) error {
   424  			switch {
   425  			case err != nil:
   426  				return err
   427  			case d.IsDir():
   428  				return nil // simply recurse into directories
   429  			case path == arg:
   430  				// non-directories given as explicit arguments are always formatted
   431  			case !isGoFilename(d.Name()):
   432  				return nil // skip walked non-Go files
   433  			}
   434  			info, err := d.Info()
   435  			if err != nil {
   436  				return err
   437  			}
   438  			s.Add(fileWeight(path, info), func(r *reporter) error {
   439  				return processFile(path, info, nil, r)
   440  			})
   441  			return nil
   442  		}); err != nil {
   443  			s.AddReport(err)
   444  		}
   445  	}
   446  }
   447  
   448  func fileWeight(path string, info fs.FileInfo) int64 {
   449  	if info == nil {
   450  		return exclusive
   451  	}
   452  	if info.Mode().Type() == fs.ModeSymlink {
   453  		var err error
   454  		info, err = os.Stat(path)
   455  		if err != nil {
   456  			return exclusive
   457  		}
   458  	}
   459  	if !info.Mode().IsRegular() {
   460  		// For non-regular files, FileInfo.Size is system-dependent and thus not a
   461  		// reliable indicator of weight.
   462  		return exclusive
   463  	}
   464  	return info.Size()
   465  }
   466  
   467  // writeFile updates a file with the new formatted data.
   468  func writeFile(filename string, orig, formatted []byte, perm fs.FileMode, size int64) error {
   469  	// Make a temporary backup file before rewriting the original file.
   470  	bakname, err := backupFile(filename, orig, perm)
   471  	if err != nil {
   472  		return err
   473  	}
   474  
   475  	fdSem <- true
   476  	defer func() { <-fdSem }()
   477  
   478  	fout, err := os.OpenFile(filename, os.O_WRONLY, perm)
   479  	if err != nil {
   480  		// We couldn't even open the file, so it should
   481  		// not have changed.
   482  		os.Remove(bakname)
   483  		return err
   484  	}
   485  	defer fout.Close() // for error paths
   486  
   487  	restoreFail := func(err error) {
   488  		fmt.Fprintf(os.Stderr, "gofmt: %s: error restoring file to original: %v; backup in %s\n", filename, err, bakname)
   489  	}
   490  
   491  	n, err := fout.Write(formatted)
   492  	if err == nil && int64(n) < size {
   493  		err = fout.Truncate(int64(n))
   494  	}
   495  
   496  	if err != nil {
   497  		// Rewriting the file failed.
   498  
   499  		if n == 0 {
   500  			// Original file unchanged.
   501  			os.Remove(bakname)
   502  			return err
   503  		}
   504  
   505  		// Try to restore the original contents.
   506  
   507  		no, erro := fout.WriteAt(orig, 0)
   508  		if erro != nil {
   509  			// That failed too.
   510  			restoreFail(erro)
   511  			return err
   512  		}
   513  
   514  		if no < n {
   515  			// Original file is shorter. Truncate.
   516  			if erro = fout.Truncate(int64(no)); erro != nil {
   517  				restoreFail(erro)
   518  				return err
   519  			}
   520  		}
   521  
   522  		if erro := fout.Close(); erro != nil {
   523  			restoreFail(erro)
   524  			return err
   525  		}
   526  
   527  		// Original contents restored.
   528  		os.Remove(bakname)
   529  		return err
   530  	}
   531  
   532  	if err := fout.Close(); err != nil {
   533  		restoreFail(err)
   534  		return err
   535  	}
   536  
   537  	// File updated.
   538  	os.Remove(bakname)
   539  	return nil
   540  }
   541  
   542  // backupFile writes data to a new file named filename<number> with permissions perm,
   543  // with <number> randomly chosen such that the file name is unique. backupFile returns
   544  // the chosen file name.
   545  func backupFile(filename string, data []byte, perm fs.FileMode) (string, error) {
   546  	fdSem <- true
   547  	defer func() { <-fdSem }()
   548  
   549  	nextRandom := func() string {
   550  		return strconv.Itoa(rand.Int())
   551  	}
   552  
   553  	dir, base := filepath.Split(filename)
   554  	var (
   555  		bakname string
   556  		f       *os.File
   557  	)
   558  	for {
   559  		bakname = filepath.Join(dir, base+"."+nextRandom())
   560  		var err error
   561  		f, err = os.OpenFile(bakname, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm)
   562  		if err == nil {
   563  			break
   564  		}
   565  		if !os.IsExist(err) {
   566  			return "", err
   567  		}
   568  	}
   569  
   570  	// write data to backup file
   571  	_, err := f.Write(data)
   572  	if err1 := f.Close(); err == nil {
   573  		err = err1
   574  	}
   575  
   576  	return bakname, err
   577  }
   578
View as plain text