vcweb.go

     1  // Copyright 2022 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package vcweb serves version control repos for testing the go command.
     6  //
     7  // It is loosely derived from golang.org/x/build/vcs-test/vcweb,
     8  // which ran as a service hosted at vcs-test.golang.org.
     9  //
    10  // When a repository URL is first requested, the vcweb [Server] dynamically
    11  // regenerates the repository using a script interpreted by a [script.Engine].
    12  // The script produces the server's contents for a corresponding root URL and
    13  // all subdirectories of that URL, which are then cached: subsequent requests
    14  // for any URL generated by the script will serve the script's previous output
    15  // until the script is modified.
    16  //
    17  // The script engine includes all of the engine's default commands and
    18  // conditions, as well as commands for each supported VCS binary (bzr, fossil,
    19  // git, hg, and svn), a "handle" command that informs the script which protocol
    20  // or handler to use to serve the request, and utilities "at" (which sets
    21  // environment variables for Git timestamps) and "unquote" (which unquotes its
    22  // argument as if it were a Go string literal).
    23  //
    24  // The server's "/" endpoint provides a summary of the available scripts,
    25  // and "/help" provides documentation for the script environment.
    26  //
    27  // To run a standalone server based on the vcweb engine, use:
    28  //
    29  //	go test cmd/go/internal/vcweb/vcstest -v --port=0
    30  package vcweb
    31  
    32  import (
    33  	"bufio"
    34  	"cmd/internal/script"
    35  	"context"
    36  	"crypto/sha256"
    37  	"errors"
    38  	"fmt"
    39  	"io"
    40  	"io/fs"
    41  	"log"
    42  	"net/http"
    43  	"os"
    44  	"os/exec"
    45  	"path"
    46  	"path/filepath"
    47  	"runtime/debug"
    48  	"strings"
    49  	"sync"
    50  	"text/tabwriter"
    51  	"time"
    52  )
    53  
    54  // A Server serves cached, dynamically-generated version control repositories.
    55  type Server struct {
    56  	env    []string
    57  	logger *log.Logger
    58  
    59  	scriptDir string
    60  	workDir   string
    61  	homeDir   string // $workdir/home
    62  	engine    *script.Engine
    63  
    64  	scriptCache sync.Map // script path → *scriptResult
    65  
    66  	vcsHandlers map[string]vcsHandler
    67  }
    68  
    69  // A vcsHandler serves repositories over HTTP for a known version-control tool.
    70  type vcsHandler interface {
    71  	Available() bool
    72  	Handler(dir string, env []string, logger *log.Logger) (http.Handler, error)
    73  }
    74  
    75  // A scriptResult describes the cached result of executing a vcweb script.
    76  type scriptResult struct {
    77  	mu sync.RWMutex
    78  
    79  	hash     [sha256.Size]byte // hash of the script file, for cache invalidation
    80  	hashTime time.Time         // timestamp at which the script was run, for diagnostics
    81  
    82  	handler http.Handler // HTTP handler configured by the script
    83  	err     error        // error from executing the script, if any
    84  }
    85  
    86  // NewServer returns a Server that generates and serves repositories in workDir
    87  // using the scripts found in scriptDir and its subdirectories.
    88  //
    89  // A request for the path /foo/bar/baz will be handled by the first script along
    90  // that path that exists: $scriptDir/foo.txt, $scriptDir/foo/bar.txt, or
    91  // $scriptDir/foo/bar/baz.txt.
    92  func NewServer(scriptDir, workDir string, logger *log.Logger) (*Server, error) {
    93  	if scriptDir == "" {
    94  		panic("vcweb.NewServer: scriptDir is required")
    95  	}
    96  	var err error
    97  	scriptDir, err = filepath.Abs(scriptDir)
    98  	if err != nil {
    99  		return nil, err
   100  	}
   101  
   102  	if workDir == "" {
   103  		workDir, err = os.MkdirTemp("", "vcweb-*")
   104  		if err != nil {
   105  			return nil, err
   106  		}
   107  		logger.Printf("vcweb work directory: %s", workDir)
   108  	} else {
   109  		workDir, err = filepath.Abs(workDir)
   110  		if err != nil {
   111  			return nil, err
   112  		}
   113  	}
   114  
   115  	homeDir := filepath.Join(workDir, "home")
   116  	if err := os.MkdirAll(homeDir, 0755); err != nil {
   117  		return nil, err
   118  	}
   119  
   120  	env := scriptEnviron(homeDir)
   121  
   122  	s := &Server{
   123  		env:       env,
   124  		logger:    logger,
   125  		scriptDir: scriptDir,
   126  		workDir:   workDir,
   127  		homeDir:   homeDir,
   128  		engine:    newScriptEngine(),
   129  		vcsHandlers: map[string]vcsHandler{
   130  			"auth":     new(authHandler),
   131  			"dir":      new(dirHandler),
   132  			"bzr":      new(bzrHandler),
   133  			"fossil":   new(fossilHandler),
   134  			"git":      new(gitHandler),
   135  			"hg":       new(hgHandler),
   136  			"insecure": new(insecureHandler),
   137  			"svn":      &svnHandler{svnRoot: workDir, logger: logger},
   138  		},
   139  	}
   140  
   141  	if err := os.WriteFile(filepath.Join(s.homeDir, ".gitconfig"), []byte(gitConfig), 0644); err != nil {
   142  		return nil, err
   143  	}
   144  	gitConfigDir := filepath.Join(s.homeDir, ".config", "git")
   145  	if err := os.MkdirAll(gitConfigDir, 0755); err != nil {
   146  		return nil, err
   147  	}
   148  	if err := os.WriteFile(filepath.Join(gitConfigDir, "ignore"), []byte(""), 0644); err != nil {
   149  		return nil, err
   150  	}
   151  
   152  	if err := os.WriteFile(filepath.Join(s.homeDir, ".hgrc"), []byte(hgrc), 0644); err != nil {
   153  		return nil, err
   154  	}
   155  
   156  	return s, nil
   157  }
   158  
   159  func (s *Server) Close() error {
   160  	var firstErr error
   161  	for _, h := range s.vcsHandlers {
   162  		if c, ok := h.(io.Closer); ok {
   163  			if closeErr := c.Close(); firstErr == nil {
   164  				firstErr = closeErr
   165  			}
   166  		}
   167  	}
   168  	return firstErr
   169  }
   170  
   171  // gitConfig contains a ~/.gitconfg file that attempts to provide
   172  // deterministic, platform-agnostic behavior for the 'git' command.
   173  var gitConfig = `
   174  [user]
   175  	name = Go Gopher
   176  	email = gopher@golang.org
   177  [init]
   178  	defaultBranch = main
   179  [core]
   180  	eol = lf
   181  [gui]
   182  	encoding = utf-8
   183  `[1:]
   184  
   185  // hgrc contains a ~/.hgrc file that attempts to provide
   186  // deterministic, platform-agnostic behavior for the 'hg' command.
   187  var hgrc = `
   188  [ui]
   189  username=Go Gopher <gopher@golang.org>
   190  [phases]
   191  new-commit=public
   192  [extensions]
   193  convert=
   194  `[1:]
   195  
   196  // ServeHTTP implements [http.Handler] for version-control repositories.
   197  func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
   198  	s.logger.Printf("serving %s", req.URL)
   199  
   200  	defer func() {
   201  		if v := recover(); v != nil {
   202  			if v == http.ErrAbortHandler {
   203  				panic(v)
   204  			}
   205  			s.logger.Fatalf("panic serving %s: %v\n%s", req.URL, v, debug.Stack())
   206  		}
   207  	}()
   208  
   209  	urlPath := req.URL.Path
   210  	if !strings.HasPrefix(urlPath, "/") {
   211  		urlPath = "/" + urlPath
   212  	}
   213  	clean := path.Clean(urlPath)[1:]
   214  	if clean == "" {
   215  		s.overview(w, req)
   216  		return
   217  	}
   218  	if clean == "help" {
   219  		s.help(w, req)
   220  		return
   221  	}
   222  
   223  	// Locate the script that generates the requested path.
   224  	// We follow directories all the way to the end, then look for a ".txt" file
   225  	// matching the first component that doesn't exist. That guarantees
   226  	// uniqueness: if a path exists as a directory, then it cannot exist as a
   227  	// ".txt" script (because the search would ignore that file).
   228  	scriptPath := "."
   229  	for part := range strings.SplitSeq(clean, "/") {
   230  		scriptPath = filepath.Join(scriptPath, part)
   231  		dir := filepath.Join(s.scriptDir, scriptPath)
   232  		if _, err := os.Stat(dir); err != nil {
   233  			if !os.IsNotExist(err) {
   234  				http.Error(w, err.Error(), http.StatusInternalServerError)
   235  				return
   236  			}
   237  			// scriptPath does not exist as a directory, so it either is the script
   238  			// location or the script doesn't exist.
   239  			break
   240  		}
   241  	}
   242  	scriptPath += ".txt"
   243  
   244  	err := s.HandleScript(scriptPath, s.logger, func(handler http.Handler) {
   245  		handler.ServeHTTP(w, req)
   246  	})
   247  	if err != nil {
   248  		s.logger.Print(err)
   249  		if _, ok := errors.AsType[ScriptNotFoundError](err); ok {
   250  			http.NotFound(w, req)
   251  		} else if _, ok := errors.AsType[ServerNotInstalledError](err); ok || errors.Is(err, exec.ErrNotFound) {
   252  			http.Error(w, err.Error(), http.StatusNotImplemented)
   253  		} else {
   254  			http.Error(w, err.Error(), http.StatusInternalServerError)
   255  		}
   256  	}
   257  }
   258  
   259  // A ScriptNotFoundError indicates that the requested script file does not exist.
   260  // (It typically wraps a "stat" error for the script file.)
   261  type ScriptNotFoundError struct{ err error }
   262  
   263  func (e ScriptNotFoundError) Error() string { return e.err.Error() }
   264  func (e ScriptNotFoundError) Unwrap() error { return e.err }
   265  
   266  // A ServerNotInstalledError indicates that the server binary required for the
   267  // indicated VCS does not exist.
   268  type ServerNotInstalledError struct{ name string }
   269  
   270  func (v ServerNotInstalledError) Error() string {
   271  	return fmt.Sprintf("server for %#q VCS is not installed", v.name)
   272  }
   273  
   274  // HandleScript ensures that the script at scriptRelPath has been evaluated
   275  // with its current contents.
   276  //
   277  // If the script completed successfully, HandleScript invokes f on the handler
   278  // with the script's result still read-locked, and waits for it to return. (That
   279  // ensures that cache invalidation does not race with an in-flight handler.)
   280  //
   281  // Otherwise, HandleScript returns the (cached) error from executing the script.
   282  func (s *Server) HandleScript(scriptRelPath string, logger *log.Logger, f func(http.Handler)) error {
   283  	ri, ok := s.scriptCache.Load(scriptRelPath)
   284  	if !ok {
   285  		ri, _ = s.scriptCache.LoadOrStore(scriptRelPath, new(scriptResult))
   286  	}
   287  	r := ri.(*scriptResult)
   288  
   289  	relDir := strings.TrimSuffix(scriptRelPath, filepath.Ext(scriptRelPath))
   290  	workDir := filepath.Join(s.workDir, relDir)
   291  	prefix := path.Join("/", filepath.ToSlash(relDir))
   292  
   293  	r.mu.RLock()
   294  	defer r.mu.RUnlock()
   295  	for {
   296  		// For efficiency, we cache the script's output (in the work directory)
   297  		// across invocations. However, to allow for rapid iteration, we hash the
   298  		// script's contents and regenerate its output if the contents change.
   299  		//
   300  		// That way, one can use 'go run main.go' in this directory to stand up a
   301  		// server and see the output of the test script in order to fine-tune it.
   302  		content, err := os.ReadFile(filepath.Join(s.scriptDir, scriptRelPath))
   303  		if err != nil {
   304  			if !os.IsNotExist(err) {
   305  				return err
   306  			}
   307  			return ScriptNotFoundError{err}
   308  		}
   309  
   310  		hash := sha256.Sum256(content)
   311  		if prevHash := r.hash; prevHash != hash {
   312  			// The script's hash has changed, so regenerate its output.
   313  			func() {
   314  				r.mu.RUnlock()
   315  				r.mu.Lock()
   316  				defer func() {
   317  					r.mu.Unlock()
   318  					r.mu.RLock()
   319  				}()
   320  				if r.hash != prevHash {
   321  					// The cached result changed while we were waiting on the lock.
   322  					// It may have been updated to our hash or something even newer,
   323  					// so don't overwrite it.
   324  					return
   325  				}
   326  
   327  				r.hash = hash
   328  				r.hashTime = time.Now()
   329  				r.handler, r.err = nil, nil
   330  
   331  				if err := os.RemoveAll(workDir); err != nil {
   332  					r.err = err
   333  					return
   334  				}
   335  
   336  				// Note: we use context.Background here instead of req.Context() so that we
   337  				// don't cache a spurious error (and lose work) if the request is canceled
   338  				// while the script is still running.
   339  				scriptHandler, err := s.loadScript(context.Background(), logger, scriptRelPath, content, workDir)
   340  				if err != nil {
   341  					r.err = err
   342  					return
   343  				}
   344  				r.handler = http.StripPrefix(prefix, scriptHandler)
   345  			}()
   346  		}
   347  
   348  		if r.hash != hash {
   349  			continue // Raced with an update from another handler; try again.
   350  		}
   351  
   352  		if r.err != nil {
   353  			return r.err
   354  		}
   355  		f(r.handler)
   356  		return nil
   357  	}
   358  }
   359  
   360  // overview serves an HTML summary of the status of the scripts in the server's
   361  // script directory.
   362  func (s *Server) overview(w http.ResponseWriter, r *http.Request) {
   363  	fmt.Fprintf(w, "<html>\n")
   364  	fmt.Fprintf(w, "<title>vcweb</title>\n<pre>\n")
   365  	fmt.Fprintf(w, "<b>vcweb</b>\n\n")
   366  	fmt.Fprintf(w, "This server serves various version control repos for testing the go command.\n\n")
   367  	fmt.Fprintf(w, "For an overview of the script language, see <a href=\"/help\">/help</a>.\n\n")
   368  
   369  	fmt.Fprintf(w, "<b>cache</b>\n")
   370  
   371  	tw := tabwriter.NewWriter(w, 1, 8, 1, '\t', 0)
   372  	err := filepath.WalkDir(s.scriptDir, func(path string, d fs.DirEntry, err error) error {
   373  		if err != nil {
   374  			return err
   375  		}
   376  		if filepath.Ext(path) != ".txt" {
   377  			return nil
   378  		}
   379  
   380  		rel, err := filepath.Rel(s.scriptDir, path)
   381  		if err != nil {
   382  			return err
   383  		}
   384  		hashTime := "(not loaded)"
   385  		status := ""
   386  		if ri, ok := s.scriptCache.Load(rel); ok {
   387  			r := ri.(*scriptResult)
   388  			r.mu.RLock()
   389  			defer r.mu.RUnlock()
   390  
   391  			if !r.hashTime.IsZero() {
   392  				hashTime = r.hashTime.Format(time.RFC3339)
   393  			}
   394  			if r.err == nil {
   395  				status = "ok"
   396  			} else {
   397  				status = r.err.Error()
   398  			}
   399  		}
   400  		fmt.Fprintf(tw, "%s\t%s\t%s\n", rel, hashTime, status)
   401  		return nil
   402  	})
   403  	tw.Flush()
   404  
   405  	if err != nil {
   406  		fmt.Fprintln(w, err)
   407  	}
   408  }
   409  
   410  // help serves a plain-text summary of the server's supported script language.
   411  func (s *Server) help(w http.ResponseWriter, req *http.Request) {
   412  	st, err := s.newState(req.Context(), s.workDir)
   413  	if err != nil {
   414  		http.Error(w, err.Error(), http.StatusInternalServerError)
   415  		return
   416  	}
   417  
   418  	scriptLog := new(strings.Builder)
   419  	err = s.engine.Execute(st, "help", bufio.NewReader(strings.NewReader("help")), scriptLog)
   420  	if err != nil {
   421  		http.Error(w, err.Error(), http.StatusInternalServerError)
   422  		return
   423  	}
   424  
   425  	w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
   426  	io.WriteString(w, scriptLog.String())
   427  }
   428
View as plain text