generate.go

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package generate implements the “go generate” command.
     6  package generate
     7  
     8  import (
     9  	"bufio"
    10  	"bytes"
    11  	"context"
    12  	"fmt"
    13  	"go/parser"
    14  	"go/token"
    15  	"io"
    16  	"log"
    17  	"os"
    18  	"os/exec"
    19  	"path/filepath"
    20  	"regexp"
    21  	"slices"
    22  	"strconv"
    23  	"strings"
    24  
    25  	"cmd/go/internal/base"
    26  	"cmd/go/internal/cfg"
    27  	"cmd/go/internal/load"
    28  	"cmd/go/internal/modload"
    29  	"cmd/go/internal/str"
    30  	"cmd/go/internal/work"
    31  	"cmd/internal/pathcache"
    32  )
    33  
    34  var CmdGenerate = &base.Command{
    35  	Run:       runGenerate,
    36  	UsageLine: "go generate [-run regexp] [-n] [-v] [-x] [build flags] [file.go... | packages]",
    37  	Short:     "generate Go files by processing source",
    38  	Long: `
    39  Generate runs commands described by directives within existing
    40  files. Those commands can run any process but the intent is to
    41  create or update Go source files.
    42  
    43  Go generate is never run automatically by go build, go test,
    44  and so on. It must be run explicitly.
    45  
    46  Go generate scans the file for directives, which are lines of
    47  the form,
    48  
    49  	//go:generate command argument...
    50  
    51  (note: no leading spaces and no space in "//go") where command
    52  is the generator to be run, corresponding to an executable file
    53  that can be run locally. It must either be in the shell path
    54  (gofmt), a fully qualified path (/usr/you/bin/mytool), or a
    55  command alias, described below.
    56  
    57  Note that go generate does not parse the file, so lines that look
    58  like directives in comments or multiline strings will be treated
    59  as directives.
    60  
    61  The arguments to the directive are space-separated tokens or
    62  double-quoted strings passed to the generator as individual
    63  arguments when it is run.
    64  
    65  Quoted strings use Go syntax and are evaluated before execution; a
    66  quoted string appears as a single argument to the generator.
    67  
    68  To convey to humans and machine tools that code is generated,
    69  generated source should have a line that matches the following
    70  regular expression (in Go syntax):
    71  
    72  	^// Code generated .* DO NOT EDIT\.$
    73  
    74  This line must appear before the first non-comment, non-blank
    75  text in the file.
    76  
    77  Go generate sets several variables when it runs the generator:
    78  
    79  	$GOARCH
    80  		The execution architecture (arm, amd64, etc.)
    81  	$GOOS
    82  		The execution operating system (linux, windows, etc.)
    83  	$GOFILE
    84  		The base name of the file.
    85  	$GOLINE
    86  		The line number of the directive in the source file.
    87  	$GOPACKAGE
    88  		The name of the package of the file containing the directive.
    89  	$GOROOT
    90  		The GOROOT directory for the 'go' command that invoked the
    91  		generator, containing the Go toolchain and standard library.
    92  	$DOLLAR
    93  		A dollar sign.
    94  	$PATH
    95  		The $PATH of the parent process, with $GOROOT/bin
    96  		placed at the beginning. This causes generators
    97  		that execute 'go' commands to use the same 'go'
    98  		as the parent 'go generate' command.
    99  
   100  Other than variable substitution and quoted-string evaluation, no
   101  special processing such as "globbing" is performed on the command
   102  line.
   103  
   104  As a last step before running the command, any invocations of any
   105  environment variables with alphanumeric names, such as $GOFILE or
   106  $HOME, are expanded throughout the command line. The syntax for
   107  variable expansion is $NAME on all operating systems. Due to the
   108  order of evaluation, variables are expanded even inside quoted
   109  strings. If the variable NAME is not set, $NAME expands to the
   110  empty string.
   111  
   112  A directive of the form,
   113  
   114  	//go:generate -command xxx args...
   115  
   116  specifies, for the remainder of this source file only, that the
   117  string xxx represents the command identified by the arguments. This
   118  can be used to create aliases or to handle multiword generators.
   119  For example,
   120  
   121  	//go:generate -command foo go tool foo
   122  
   123  specifies that the command "foo" represents the generator
   124  "go tool foo".
   125  
   126  Generate processes packages in the order given on the command line,
   127  one at a time. If the command line lists .go files from a single directory,
   128  they are treated as a single package. Within a package, generate processes the
   129  source files in a package in file name order, one at a time. Within
   130  a source file, generate runs generators in the order they appear
   131  in the file, one at a time. The go generate tool also sets the build
   132  tag "generate" so that files may be examined by go generate but ignored
   133  during build.
   134  
   135  For packages with invalid code, generate processes only source files with a
   136  valid package clause.
   137  
   138  If any generator returns an error exit status, "go generate" skips
   139  all further processing for that package.
   140  
   141  The generator is run in the package's source directory.
   142  
   143  Go generate accepts two specific flags:
   144  
   145  	-run=""
   146  		if non-empty, specifies a regular expression to select
   147  		directives whose full original source text (excluding
   148  		any trailing spaces and final newline) matches the
   149  		expression.
   150  
   151  	-skip=""
   152  		if non-empty, specifies a regular expression to suppress
   153  		directives whose full original source text (excluding
   154  		any trailing spaces and final newline) matches the
   155  		expression. If a directive matches both the -run and
   156  		the -skip arguments, it is skipped.
   157  
   158  It also accepts the standard build flags including -v, -n, and -x.
   159  The -v flag prints the names of packages and files as they are
   160  processed.
   161  The -n flag prints commands that would be executed.
   162  The -x flag prints commands as they are executed.
   163  
   164  For more about build flags, see 'go help build'.
   165  
   166  For more about specifying packages, see 'go help packages'.
   167  	`,
   168  }
   169  
   170  var (
   171  	generateRunFlag string         // generate -run flag
   172  	generateRunRE   *regexp.Regexp // compiled expression for -run
   173  
   174  	generateSkipFlag string         // generate -skip flag
   175  	generateSkipRE   *regexp.Regexp // compiled expression for -skip
   176  )
   177  
   178  func init() {
   179  	work.AddBuildFlags(CmdGenerate, work.OmitBuildOnlyFlags)
   180  	CmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "")
   181  	CmdGenerate.Flag.StringVar(&generateSkipFlag, "skip", "", "")
   182  }
   183  
   184  func runGenerate(ctx context.Context, cmd *base.Command, args []string) {
   185  	moduleLoaderState := modload.NewState()
   186  	moduleLoaderState.InitWorkfile()
   187  
   188  	if generateRunFlag != "" {
   189  		var err error
   190  		generateRunRE, err = regexp.Compile(generateRunFlag)
   191  		if err != nil {
   192  			log.Fatalf("generate: %s", err)
   193  		}
   194  	}
   195  	if generateSkipFlag != "" {
   196  		var err error
   197  		generateSkipRE, err = regexp.Compile(generateSkipFlag)
   198  		if err != nil {
   199  			log.Fatalf("generate: %s", err)
   200  		}
   201  	}
   202  
   203  	cfg.BuildContext.BuildTags = append(cfg.BuildContext.BuildTags, "generate")
   204  
   205  	// Even if the arguments are .go files, this loop suffices.
   206  	printed := false
   207  	pkgOpts := load.PackageOpts{IgnoreImports: true}
   208  	for _, pkg := range load.PackagesAndErrors(moduleLoaderState, ctx, pkgOpts, args) {
   209  		if moduleLoaderState.Enabled() && pkg.Module != nil && !pkg.Module.Main {
   210  			if !printed {
   211  				fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n")
   212  				printed = true
   213  			}
   214  			continue
   215  		}
   216  
   217  		if pkg.Error != nil && len(pkg.InternalAllGoFiles()) == 0 {
   218  			// A directory only contains a Go package if it has at least
   219  			// one .go source file, so the fact that there are no files
   220  			// implies that the package couldn't be found.
   221  			base.Errorf("%v", pkg.Error)
   222  		}
   223  
   224  		for _, file := range pkg.InternalGoFiles() {
   225  			if !generate(file) {
   226  				break
   227  			}
   228  		}
   229  
   230  		for _, file := range pkg.InternalXGoFiles() {
   231  			if !generate(file) {
   232  				break
   233  			}
   234  		}
   235  	}
   236  	base.ExitIfErrors()
   237  }
   238  
   239  // generate runs the generation directives for a single file.
   240  func generate(absFile string) bool {
   241  	src, err := os.ReadFile(absFile)
   242  	if err != nil {
   243  		log.Fatalf("generate: %s", err)
   244  	}
   245  
   246  	// Parse package clause
   247  	filePkg, err := parser.ParseFile(token.NewFileSet(), "", src, parser.PackageClauseOnly)
   248  	if err != nil {
   249  		// Invalid package clause - ignore file.
   250  		return true
   251  	}
   252  
   253  	g := &Generator{
   254  		r:        bytes.NewReader(src),
   255  		path:     absFile,
   256  		pkg:      filePkg.Name.String(),
   257  		commands: make(map[string][]string),
   258  	}
   259  	return g.run()
   260  }
   261  
   262  // A Generator represents the state of a single Go source file
   263  // being scanned for generator commands.
   264  type Generator struct {
   265  	r        io.Reader
   266  	path     string // full rooted path name.
   267  	dir      string // full rooted directory of file.
   268  	file     string // base name of file.
   269  	pkg      string
   270  	commands map[string][]string
   271  	lineNum  int // current line number.
   272  	env      []string
   273  }
   274  
   275  // run runs the generators in the current file.
   276  func (g *Generator) run() (ok bool) {
   277  	// Processing below here calls g.errorf on failure, which does panic(stop).
   278  	// If we encounter an error, we abort the package.
   279  	defer func() {
   280  		e := recover()
   281  		if e != nil {
   282  			ok = false
   283  			if e != stop {
   284  				panic(e)
   285  			}
   286  			base.SetExitStatus(1)
   287  		}
   288  	}()
   289  	g.dir, g.file = filepath.Split(g.path)
   290  	g.dir = filepath.Clean(g.dir) // No final separator please.
   291  	if cfg.BuildV {
   292  		fmt.Fprintf(os.Stderr, "%s\n", base.ShortPath(g.path))
   293  	}
   294  
   295  	// Scan for lines that start "//go:generate".
   296  	// Can't use bufio.Scanner because it can't handle long lines,
   297  	// which are likely to appear when using generate.
   298  	input := bufio.NewReader(g.r)
   299  	var err error
   300  	// One line per loop.
   301  	for {
   302  		g.lineNum++ // 1-indexed.
   303  		var buf []byte
   304  		buf, err = input.ReadSlice('\n')
   305  		if err == bufio.ErrBufferFull {
   306  			// Line too long - consume and ignore.
   307  			if isGoGenerate(buf) {
   308  				g.errorf("directive too long")
   309  			}
   310  			for err == bufio.ErrBufferFull {
   311  				_, err = input.ReadSlice('\n')
   312  			}
   313  			if err != nil {
   314  				break
   315  			}
   316  			continue
   317  		}
   318  
   319  		if err != nil {
   320  			// Check for marker at EOF without final \n.
   321  			if err == io.EOF && isGoGenerate(buf) {
   322  				err = io.ErrUnexpectedEOF
   323  			}
   324  			break
   325  		}
   326  
   327  		if !isGoGenerate(buf) {
   328  			continue
   329  		}
   330  		if generateRunFlag != "" && !generateRunRE.Match(bytes.TrimSpace(buf)) {
   331  			continue
   332  		}
   333  		if generateSkipFlag != "" && generateSkipRE.Match(bytes.TrimSpace(buf)) {
   334  			continue
   335  		}
   336  
   337  		g.setEnv()
   338  		words := g.split(string(buf))
   339  		if len(words) == 0 {
   340  			g.errorf("no arguments to directive")
   341  		}
   342  		if words[0] == "-command" {
   343  			g.setShorthand(words)
   344  			continue
   345  		}
   346  		// Run the command line.
   347  		if cfg.BuildN || cfg.BuildX {
   348  			fmt.Fprintf(os.Stderr, "%s\n", strings.Join(words, " "))
   349  		}
   350  		if cfg.BuildN {
   351  			continue
   352  		}
   353  		g.exec(words)
   354  	}
   355  	if err != nil && err != io.EOF {
   356  		g.errorf("error reading %s: %s", base.ShortPath(g.path), err)
   357  	}
   358  	return true
   359  }
   360  
   361  func isGoGenerate(buf []byte) bool {
   362  	return bytes.HasPrefix(buf, []byte("//go:generate ")) || bytes.HasPrefix(buf, []byte("//go:generate\t"))
   363  }
   364  
   365  // setEnv sets the extra environment variables used when executing a
   366  // single go:generate command.
   367  func (g *Generator) setEnv() {
   368  	env := []string{
   369  		"GOROOT=" + cfg.GOROOT,
   370  		"GOARCH=" + cfg.BuildContext.GOARCH,
   371  		"GOOS=" + cfg.BuildContext.GOOS,
   372  		"GOFILE=" + g.file,
   373  		"GOLINE=" + strconv.Itoa(g.lineNum),
   374  		"GOPACKAGE=" + g.pkg,
   375  		"DOLLAR=" + "$",
   376  	}
   377  	env = base.AppendPATH(env)
   378  	env = base.AppendPWD(env, g.dir)
   379  	g.env = env
   380  }
   381  
   382  // split breaks the line into words, evaluating quoted
   383  // strings and evaluating environment variables.
   384  // The initial //go:generate element is present in line.
   385  func (g *Generator) split(line string) []string {
   386  	// Parse line, obeying quoted strings.
   387  	var words []string
   388  	line = line[len("//go:generate ") : len(line)-1] // Drop preamble and final newline.
   389  	// There may still be a carriage return.
   390  	if len(line) > 0 && line[len(line)-1] == '\r' {
   391  		line = line[:len(line)-1]
   392  	}
   393  	// One (possibly quoted) word per iteration.
   394  Words:
   395  	for {
   396  		line = strings.TrimLeft(line, " \t")
   397  		if len(line) == 0 {
   398  			break
   399  		}
   400  		if line[0] == '"' {
   401  			for i := 1; i < len(line); i++ {
   402  				c := line[i] // Only looking for ASCII so this is OK.
   403  				switch c {
   404  				case '\\':
   405  					if i+1 == len(line) {
   406  						g.errorf("bad backslash")
   407  					}
   408  					i++ // Absorb next byte (If it's a multibyte we'll get an error in Unquote).
   409  				case '"':
   410  					word, err := strconv.Unquote(line[0 : i+1])
   411  					if err != nil {
   412  						g.errorf("bad quoted string")
   413  					}
   414  					words = append(words, word)
   415  					line = line[i+1:]
   416  					// Check the next character is space or end of line.
   417  					if len(line) > 0 && line[0] != ' ' && line[0] != '\t' {
   418  						g.errorf("expect space after quoted argument")
   419  					}
   420  					continue Words
   421  				}
   422  			}
   423  			g.errorf("mismatched quoted string")
   424  		}
   425  		i := strings.IndexAny(line, " \t")
   426  		if i < 0 {
   427  			i = len(line)
   428  		}
   429  		words = append(words, line[0:i])
   430  		line = line[i:]
   431  	}
   432  	// Substitute command if required.
   433  	if len(words) > 0 && g.commands[words[0]] != nil {
   434  		// Replace 0th word by command substitution.
   435  		//
   436  		// Force a copy of the command definition to
   437  		// ensure words doesn't end up as a reference
   438  		// to the g.commands content.
   439  		tmpCmdWords := append([]string(nil), (g.commands[words[0]])...)
   440  		words = append(tmpCmdWords, words[1:]...)
   441  	}
   442  	// Substitute environment variables.
   443  	for i, word := range words {
   444  		words[i] = os.Expand(word, g.expandVar)
   445  	}
   446  	return words
   447  }
   448  
   449  var stop = fmt.Errorf("error in generation")
   450  
   451  // errorf logs an error message prefixed with the file and line number.
   452  // It then exits the program (with exit status 1) because generation stops
   453  // at the first error.
   454  func (g *Generator) errorf(format string, args ...any) {
   455  	fmt.Fprintf(os.Stderr, "%s:%d: %s\n", base.ShortPath(g.path), g.lineNum,
   456  		fmt.Sprintf(format, args...))
   457  	panic(stop)
   458  }
   459  
   460  // expandVar expands the $XXX invocation in word. It is called
   461  // by os.Expand.
   462  func (g *Generator) expandVar(word string) string {
   463  	w := word + "="
   464  	for _, e := range g.env {
   465  		if strings.HasPrefix(e, w) {
   466  			return e[len(w):]
   467  		}
   468  	}
   469  	return os.Getenv(word)
   470  }
   471  
   472  // setShorthand installs a new shorthand as defined by a -command directive.
   473  func (g *Generator) setShorthand(words []string) {
   474  	// Create command shorthand.
   475  	if len(words) == 1 {
   476  		g.errorf("no command specified for -command")
   477  	}
   478  	command := words[1]
   479  	if g.commands[command] != nil {
   480  		g.errorf("command %q multiply defined", command)
   481  	}
   482  	g.commands[command] = slices.Clip(words[2:])
   483  }
   484  
   485  // exec runs the command specified by the argument. The first word is
   486  // the command name itself.
   487  func (g *Generator) exec(words []string) {
   488  	path := words[0]
   489  	if path != "" && !strings.Contains(path, string(os.PathSeparator)) {
   490  		// If a generator says '//go:generate go run <blah>' it almost certainly
   491  		// intends to use the same 'go' as 'go generate' itself.
   492  		// Prefer to resolve the binary from GOROOT/bin, and for consistency
   493  		// prefer to resolve any other commands there too.
   494  		gorootBinPath, err := pathcache.LookPath(filepath.Join(cfg.GOROOTbin, path))
   495  		if err == nil {
   496  			path = gorootBinPath
   497  		}
   498  	}
   499  	cmd := exec.Command(path, words[1:]...)
   500  	cmd.Args[0] = words[0] // Overwrite with the original in case it was rewritten above.
   501  
   502  	// Standard in and out of generator should be the usual.
   503  	cmd.Stdout = os.Stdout
   504  	cmd.Stderr = os.Stderr
   505  	// Run the command in the package directory.
   506  	cmd.Dir = g.dir
   507  	cmd.Env = str.StringList(cfg.OrigEnv, g.env)
   508  	err := cmd.Run()
   509  	if err != nil {
   510  		g.errorf("running %q: %s", words[0], err)
   511  	}
   512  }
   513
View as plain text