vcweb.go

     1  // Copyright 2022 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package vcweb serves version control repos for testing the go command.
     6  //
     7  // It is loosely derived from golang.org/x/build/vcs-test/vcweb,
     8  // which ran as a service hosted at vcs-test.golang.org.
     9  //
    10  // When a repository URL is first requested, the vcweb [Server] dynamically
    11  // regenerates the repository using a script interpreted by a [script.Engine].
    12  // The script produces the server's contents for a corresponding root URL and
    13  // all subdirectories of that URL, which are then cached: subsequent requests
    14  // for any URL generated by the script will serve the script's previous output
    15  // until the script is modified.
    16  //
    17  // The script engine includes all of the engine's default commands and
    18  // conditions, as well as commands for each supported VCS binary (fossil,
    19  // git, hg, and svn), a "handle" command that informs the script which protocol
    20  // or handler to use to serve the request, and utilities "at" (which sets
    21  // environment variables for Git timestamps) and "unquote" (which unquotes its
    22  // argument as if it were a Go string literal).
    23  //
    24  // The server's "/" endpoint provides a summary of the available scripts,
    25  // and "/help" provides documentation for the script environment.
    26  //
    27  // To run a standalone server based on the vcweb engine, use:
    28  //
    29  //	go test cmd/go/internal/vcweb/vcstest -v --port=0
    30  package vcweb
    31  
    32  import (
    33  	"bufio"
    34  	"cmd/internal/script"
    35  	"context"
    36  	"crypto/sha256"
    37  	"errors"
    38  	"fmt"
    39  	"io"
    40  	"io/fs"
    41  	"log"
    42  	"net/http"
    43  	"os"
    44  	"os/exec"
    45  	"path"
    46  	"path/filepath"
    47  	"runtime/debug"
    48  	"strings"
    49  	"sync"
    50  	"text/tabwriter"
    51  	"time"
    52  )
    53  
    54  // A Server serves cached, dynamically-generated version control repositories.
    55  type Server struct {
    56  	env    []string
    57  	logger *log.Logger
    58  
    59  	scriptDir string
    60  	workDir   string
    61  	homeDir   string // $workdir/home
    62  	engine    *script.Engine
    63  
    64  	scriptCache sync.Map // script path → *scriptResult
    65  
    66  	vcsHandlers map[string]vcsHandler
    67  }
    68  
    69  // A vcsHandler serves repositories over HTTP for a known version-control tool.
    70  type vcsHandler interface {
    71  	Available() bool
    72  	Handler(dir string, env []string, logger *log.Logger) (http.Handler, error)
    73  }
    74  
    75  // A scriptResult describes the cached result of executing a vcweb script.
    76  type scriptResult struct {
    77  	mu sync.RWMutex
    78  
    79  	hash     [sha256.Size]byte // hash of the script file, for cache invalidation
    80  	hashTime time.Time         // timestamp at which the script was run, for diagnostics
    81  
    82  	handler http.Handler // HTTP handler configured by the script
    83  	err     error        // error from executing the script, if any
    84  }
    85  
    86  // NewServer returns a Server that generates and serves repositories in workDir
    87  // using the scripts found in scriptDir and its subdirectories.
    88  //
    89  // A request for the path /foo/bar/baz will be handled by the first script along
    90  // that path that exists: $scriptDir/foo.txt, $scriptDir/foo/bar.txt, or
    91  // $scriptDir/foo/bar/baz.txt.
    92  func NewServer(scriptDir, workDir string, logger *log.Logger) (*Server, error) {
    93  	if scriptDir == "" {
    94  		panic("vcweb.NewServer: scriptDir is required")
    95  	}
    96  	var err error
    97  	scriptDir, err = filepath.Abs(scriptDir)
    98  	if err != nil {
    99  		return nil, err
   100  	}
   101  
   102  	if workDir == "" {
   103  		workDir, err = os.MkdirTemp("", "vcweb-*")
   104  		if err != nil {
   105  			return nil, err
   106  		}
   107  		logger.Printf("vcweb work directory: %s", workDir)
   108  	} else {
   109  		workDir, err = filepath.Abs(workDir)
   110  		if err != nil {
   111  			return nil, err
   112  		}
   113  	}
   114  
   115  	homeDir := filepath.Join(workDir, "home")
   116  	if err := os.MkdirAll(homeDir, 0755); err != nil {
   117  		return nil, err
   118  	}
   119  
   120  	env := scriptEnviron(homeDir)
   121  
   122  	s := &Server{
   123  		env:       env,
   124  		logger:    logger,
   125  		scriptDir: scriptDir,
   126  		workDir:   workDir,
   127  		homeDir:   homeDir,
   128  		engine:    newScriptEngine(),
   129  		vcsHandlers: map[string]vcsHandler{
   130  			"auth":     new(authHandler),
   131  			"dir":      new(dirHandler),
   132  			"fossil":   new(fossilHandler),
   133  			"git":      new(gitHandler),
   134  			"hg":       new(hgHandler),
   135  			"insecure": new(insecureHandler),
   136  			"svn":      &svnHandler{svnRoot: workDir, logger: logger},
   137  		},
   138  	}
   139  
   140  	if err := os.WriteFile(filepath.Join(s.homeDir, ".gitconfig"), []byte(gitConfig), 0644); err != nil {
   141  		return nil, err
   142  	}
   143  	gitConfigDir := filepath.Join(s.homeDir, ".config", "git")
   144  	if err := os.MkdirAll(gitConfigDir, 0755); err != nil {
   145  		return nil, err
   146  	}
   147  	if err := os.WriteFile(filepath.Join(gitConfigDir, "ignore"), []byte(""), 0644); err != nil {
   148  		return nil, err
   149  	}
   150  
   151  	if err := os.WriteFile(filepath.Join(s.homeDir, ".hgrc"), []byte(hgrc), 0644); err != nil {
   152  		return nil, err
   153  	}
   154  
   155  	return s, nil
   156  }
   157  
   158  func (s *Server) Close() error {
   159  	var firstErr error
   160  	for _, h := range s.vcsHandlers {
   161  		if c, ok := h.(io.Closer); ok {
   162  			if closeErr := c.Close(); firstErr == nil {
   163  				firstErr = closeErr
   164  			}
   165  		}
   166  	}
   167  	return firstErr
   168  }
   169  
   170  // gitConfig contains a ~/.gitconfg file that attempts to provide
   171  // deterministic, platform-agnostic behavior for the 'git' command.
   172  var gitConfig = `
   173  [user]
   174  	name = Go Gopher
   175  	email = gopher@golang.org
   176  [init]
   177  	defaultBranch = main
   178  [core]
   179  	eol = lf
   180  [gui]
   181  	encoding = utf-8
   182  `[1:]
   183  
   184  // hgrc contains a ~/.hgrc file that attempts to provide
   185  // deterministic, platform-agnostic behavior for the 'hg' command.
   186  var hgrc = `
   187  [ui]
   188  username=Go Gopher <gopher@golang.org>
   189  [phases]
   190  new-commit=public
   191  [extensions]
   192  convert=
   193  `[1:]
   194  
   195  // ServeHTTP implements [http.Handler] for version-control repositories.
   196  func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
   197  	s.logger.Printf("serving %s", req.URL)
   198  
   199  	defer func() {
   200  		if v := recover(); v != nil {
   201  			if v == http.ErrAbortHandler {
   202  				panic(v)
   203  			}
   204  			s.logger.Fatalf("panic serving %s: %v\n%s", req.URL, v, debug.Stack())
   205  		}
   206  	}()
   207  
   208  	urlPath := req.URL.Path
   209  	if !strings.HasPrefix(urlPath, "/") {
   210  		urlPath = "/" + urlPath
   211  	}
   212  	clean := path.Clean(urlPath)[1:]
   213  	if clean == "" {
   214  		s.overview(w, req)
   215  		return
   216  	}
   217  	if clean == "help" {
   218  		s.help(w, req)
   219  		return
   220  	}
   221  
   222  	// Locate the script that generates the requested path.
   223  	// We follow directories all the way to the end, then look for a ".txt" file
   224  	// matching the first component that doesn't exist. That guarantees
   225  	// uniqueness: if a path exists as a directory, then it cannot exist as a
   226  	// ".txt" script (because the search would ignore that file).
   227  	scriptPath := "."
   228  	for part := range strings.SplitSeq(clean, "/") {
   229  		scriptPath = filepath.Join(scriptPath, part)
   230  		dir := filepath.Join(s.scriptDir, scriptPath)
   231  		if _, err := os.Stat(dir); err != nil {
   232  			if !os.IsNotExist(err) {
   233  				http.Error(w, err.Error(), http.StatusInternalServerError)
   234  				return
   235  			}
   236  			// scriptPath does not exist as a directory, so it either is the script
   237  			// location or the script doesn't exist.
   238  			break
   239  		}
   240  	}
   241  	scriptPath += ".txt"
   242  
   243  	err := s.HandleScript(scriptPath, s.logger, func(handler http.Handler) {
   244  		handler.ServeHTTP(w, req)
   245  	})
   246  	if err != nil {
   247  		s.logger.Print(err)
   248  		if _, ok := errors.AsType[ScriptNotFoundError](err); ok {
   249  			http.NotFound(w, req)
   250  		} else if _, ok := errors.AsType[ServerNotInstalledError](err); ok || errors.Is(err, exec.ErrNotFound) {
   251  			http.Error(w, err.Error(), http.StatusNotImplemented)
   252  		} else {
   253  			http.Error(w, err.Error(), http.StatusInternalServerError)
   254  		}
   255  	}
   256  }
   257  
   258  // A ScriptNotFoundError indicates that the requested script file does not exist.
   259  // (It typically wraps a "stat" error for the script file.)
   260  type ScriptNotFoundError struct{ err error }
   261  
   262  func (e ScriptNotFoundError) Error() string { return e.err.Error() }
   263  func (e ScriptNotFoundError) Unwrap() error { return e.err }
   264  
   265  // A ServerNotInstalledError indicates that the server binary required for the
   266  // indicated VCS does not exist.
   267  type ServerNotInstalledError struct{ name string }
   268  
   269  func (v ServerNotInstalledError) Error() string {
   270  	return fmt.Sprintf("server for %#q VCS is not installed", v.name)
   271  }
   272  
   273  // HandleScript ensures that the script at scriptRelPath has been evaluated
   274  // with its current contents.
   275  //
   276  // If the script completed successfully, HandleScript invokes f on the handler
   277  // with the script's result still read-locked, and waits for it to return. (That
   278  // ensures that cache invalidation does not race with an in-flight handler.)
   279  //
   280  // Otherwise, HandleScript returns the (cached) error from executing the script.
   281  func (s *Server) HandleScript(scriptRelPath string, logger *log.Logger, f func(http.Handler)) error {
   282  	ri, ok := s.scriptCache.Load(scriptRelPath)
   283  	if !ok {
   284  		ri, _ = s.scriptCache.LoadOrStore(scriptRelPath, new(scriptResult))
   285  	}
   286  	r := ri.(*scriptResult)
   287  
   288  	relDir := strings.TrimSuffix(scriptRelPath, filepath.Ext(scriptRelPath))
   289  	workDir := filepath.Join(s.workDir, relDir)
   290  	prefix := path.Join("/", filepath.ToSlash(relDir))
   291  
   292  	r.mu.RLock()
   293  	defer r.mu.RUnlock()
   294  	for {
   295  		// For efficiency, we cache the script's output (in the work directory)
   296  		// across invocations. However, to allow for rapid iteration, we hash the
   297  		// script's contents and regenerate its output if the contents change.
   298  		//
   299  		// That way, one can use 'go run main.go' in this directory to stand up a
   300  		// server and see the output of the test script in order to fine-tune it.
   301  		content, err := os.ReadFile(filepath.Join(s.scriptDir, scriptRelPath))
   302  		if err != nil {
   303  			if !os.IsNotExist(err) {
   304  				return err
   305  			}
   306  			return ScriptNotFoundError{err}
   307  		}
   308  
   309  		hash := sha256.Sum256(content)
   310  		if prevHash := r.hash; prevHash != hash {
   311  			// The script's hash has changed, so regenerate its output.
   312  			func() {
   313  				r.mu.RUnlock()
   314  				r.mu.Lock()
   315  				defer func() {
   316  					r.mu.Unlock()
   317  					r.mu.RLock()
   318  				}()
   319  				if r.hash != prevHash {
   320  					// The cached result changed while we were waiting on the lock.
   321  					// It may have been updated to our hash or something even newer,
   322  					// so don't overwrite it.
   323  					return
   324  				}
   325  
   326  				r.hash = hash
   327  				r.hashTime = time.Now()
   328  				r.handler, r.err = nil, nil
   329  
   330  				if err := os.RemoveAll(workDir); err != nil {
   331  					r.err = err
   332  					return
   333  				}
   334  
   335  				// Note: we use context.Background here instead of req.Context() so that we
   336  				// don't cache a spurious error (and lose work) if the request is canceled
   337  				// while the script is still running.
   338  				scriptHandler, err := s.loadScript(context.Background(), logger, scriptRelPath, content, workDir)
   339  				if err != nil {
   340  					r.err = err
   341  					return
   342  				}
   343  				r.handler = http.StripPrefix(prefix, scriptHandler)
   344  			}()
   345  		}
   346  
   347  		if r.hash != hash {
   348  			continue // Raced with an update from another handler; try again.
   349  		}
   350  
   351  		if r.err != nil {
   352  			return r.err
   353  		}
   354  		f(r.handler)
   355  		return nil
   356  	}
   357  }
   358  
   359  // overview serves an HTML summary of the status of the scripts in the server's
   360  // script directory.
   361  func (s *Server) overview(w http.ResponseWriter, r *http.Request) {
   362  	fmt.Fprintf(w, "<html>\n")
   363  	fmt.Fprintf(w, "<title>vcweb</title>\n<pre>\n")
   364  	fmt.Fprintf(w, "<b>vcweb</b>\n\n")
   365  	fmt.Fprintf(w, "This server serves various version control repos for testing the go command.\n\n")
   366  	fmt.Fprintf(w, "For an overview of the script language, see <a href=\"/help\">/help</a>.\n\n")
   367  
   368  	fmt.Fprintf(w, "<b>cache</b>\n")
   369  
   370  	tw := tabwriter.NewWriter(w, 1, 8, 1, '\t', 0)
   371  	err := filepath.WalkDir(s.scriptDir, func(path string, d fs.DirEntry, err error) error {
   372  		if err != nil {
   373  			return err
   374  		}
   375  		if filepath.Ext(path) != ".txt" {
   376  			return nil
   377  		}
   378  
   379  		rel, err := filepath.Rel(s.scriptDir, path)
   380  		if err != nil {
   381  			return err
   382  		}
   383  		hashTime := "(not loaded)"
   384  		status := ""
   385  		if ri, ok := s.scriptCache.Load(rel); ok {
   386  			r := ri.(*scriptResult)
   387  			r.mu.RLock()
   388  			defer r.mu.RUnlock()
   389  
   390  			if !r.hashTime.IsZero() {
   391  				hashTime = r.hashTime.Format(time.RFC3339)
   392  			}
   393  			if r.err == nil {
   394  				status = "ok"
   395  			} else {
   396  				status = r.err.Error()
   397  			}
   398  		}
   399  		fmt.Fprintf(tw, "%s\t%s\t%s\n", rel, hashTime, status)
   400  		return nil
   401  	})
   402  	tw.Flush()
   403  
   404  	if err != nil {
   405  		fmt.Fprintln(w, err)
   406  	}
   407  }
   408  
   409  // help serves a plain-text summary of the server's supported script language.
   410  func (s *Server) help(w http.ResponseWriter, req *http.Request) {
   411  	st, err := s.newState(req.Context(), s.workDir)
   412  	if err != nil {
   413  		http.Error(w, err.Error(), http.StatusInternalServerError)
   414  		return
   415  	}
   416  
   417  	scriptLog := new(strings.Builder)
   418  	err = s.engine.Execute(st, "help", bufio.NewReader(strings.NewReader("help")), scriptLog)
   419  	if err != nil {
   420  		http.Error(w, err.Error(), http.StatusInternalServerError)
   421  		return
   422  	}
   423  
   424  	w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
   425  	io.WriteString(w, scriptLog.String())
   426  }
   427
View as plain text