script_test.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Script-driven tests.
     6  // See testdata/script/README for an overview.
     7  
     8  //go:generate go test cmd/go -v -run=TestScript/README --fixreadme
     9  
    10  package main_test
    11  
    12  import (
    13  	"bufio"
    14  	"bytes"
    15  	"context"
    16  	_ "embed"
    17  	"flag"
    18  	"internal/testenv"
    19  	"internal/txtar"
    20  	"net/url"
    21  	"os"
    22  	"path/filepath"
    23  	"runtime"
    24  	"strings"
    25  	"testing"
    26  	"time"
    27  
    28  	"cmd/go/internal/cfg"
    29  	"cmd/go/internal/gover"
    30  	"cmd/go/internal/script"
    31  	"cmd/go/internal/script/scripttest"
    32  	"cmd/go/internal/vcweb/vcstest"
    33  
    34  	"golang.org/x/telemetry/counter/countertest"
    35  )
    36  
    37  var testSum = flag.String("testsum", "", `may be tidy, listm, or listall. If set, TestScript generates a go.sum file at the beginning of each test and updates test files if they pass.`)
    38  
    39  // TestScript runs the tests in testdata/script/*.txt.
    40  func TestScript(t *testing.T) {
    41  	testenv.MustHaveGoBuild(t)
    42  	testenv.SkipIfShortAndSlow(t)
    43  
    44  	srv, err := vcstest.NewServer()
    45  	if err != nil {
    46  		t.Fatal(err)
    47  	}
    48  	t.Cleanup(func() {
    49  		if err := srv.Close(); err != nil {
    50  			t.Fatal(err)
    51  		}
    52  	})
    53  	certFile, err := srv.WriteCertificateFile()
    54  	if err != nil {
    55  		t.Fatal(err)
    56  	}
    57  
    58  	StartProxy()
    59  
    60  	var (
    61  		ctx         = context.Background()
    62  		gracePeriod = 100 * time.Millisecond
    63  	)
    64  	if deadline, ok := t.Deadline(); ok {
    65  		timeout := time.Until(deadline)
    66  
    67  		// If time allows, increase the termination grace period to 5% of the
    68  		// remaining time.
    69  		if gp := timeout / 20; gp > gracePeriod {
    70  			gracePeriod = gp
    71  		}
    72  
    73  		// When we run commands that execute subprocesses, we want to reserve two
    74  		// grace periods to clean up. We will send the first termination signal when
    75  		// the context expires, then wait one grace period for the process to
    76  		// produce whatever useful output it can (such as a stack trace). After the
    77  		// first grace period expires, we'll escalate to os.Kill, leaving the second
    78  		// grace period for the test function to record its output before the test
    79  		// process itself terminates.
    80  		timeout -= 2 * gracePeriod
    81  
    82  		var cancel context.CancelFunc
    83  		ctx, cancel = context.WithTimeout(ctx, timeout)
    84  		t.Cleanup(cancel)
    85  	}
    86  
    87  	env, err := scriptEnv(srv, certFile)
    88  	if err != nil {
    89  		t.Fatal(err)
    90  	}
    91  	engine := &script.Engine{
    92  		Conds: scriptConditions(),
    93  		Cmds:  scriptCommands(quitSignal(), gracePeriod),
    94  		Quiet: !testing.Verbose(),
    95  	}
    96  
    97  	t.Run("README", func(t *testing.T) {
    98  		checkScriptReadme(t, engine, env)
    99  	})
   100  
   101  	files, err := filepath.Glob("testdata/script/*.txt")
   102  	if err != nil {
   103  		t.Fatal(err)
   104  	}
   105  	for _, file := range files {
   106  		file := file
   107  		name := strings.TrimSuffix(filepath.Base(file), ".txt")
   108  		t.Run(name, func(t *testing.T) {
   109  			t.Parallel()
   110  			StartProxy()
   111  
   112  			workdir, err := os.MkdirTemp(testTmpDir, name)
   113  			if err != nil {
   114  				t.Fatal(err)
   115  			}
   116  			if !*testWork {
   117  				defer removeAll(workdir)
   118  			}
   119  
   120  			s, err := script.NewState(tbContext(ctx, t), workdir, env)
   121  			if err != nil {
   122  				t.Fatal(err)
   123  			}
   124  
   125  			// Unpack archive.
   126  			a, err := txtar.ParseFile(file)
   127  			if err != nil {
   128  				t.Fatal(err)
   129  			}
   130  			telemetryDir := initScriptDirs(t, s)
   131  			if err := s.ExtractFiles(a); err != nil {
   132  				t.Fatal(err)
   133  			}
   134  
   135  			t.Log(time.Now().UTC().Format(time.RFC3339))
   136  			work, _ := s.LookupEnv("WORK")
   137  			t.Logf("$WORK=%s", work)
   138  
   139  			// With -testsum, if a go.mod file is present in the test's initial
   140  			// working directory, run 'go mod tidy'.
   141  			if *testSum != "" {
   142  				if updateSum(t, engine, s, a) {
   143  					defer func() {
   144  						if t.Failed() {
   145  							return
   146  						}
   147  						data := txtar.Format(a)
   148  						if err := os.WriteFile(file, data, 0666); err != nil {
   149  							t.Errorf("rewriting test file: %v", err)
   150  						}
   151  					}()
   152  				}
   153  			}
   154  
   155  			// Note: Do not use filepath.Base(file) here:
   156  			// editors that can jump to file:line references in the output
   157  			// will work better seeing the full path relative to cmd/go
   158  			// (where the "go test" command is usually run).
   159  			scripttest.Run(t, engine, s, file, bytes.NewReader(a.Comment))
   160  			checkCounters(t, telemetryDir)
   161  		})
   162  	}
   163  }
   164  
   165  // testingTBKey is the Context key for a testing.TB.
   166  type testingTBKey struct{}
   167  
   168  // tbContext returns a Context derived from ctx and associated with t.
   169  func tbContext(ctx context.Context, t testing.TB) context.Context {
   170  	return context.WithValue(ctx, testingTBKey{}, t)
   171  }
   172  
   173  // tbFromContext returns the testing.TB associated with ctx, if any.
   174  func tbFromContext(ctx context.Context) (testing.TB, bool) {
   175  	t := ctx.Value(testingTBKey{})
   176  	if t == nil {
   177  		return nil, false
   178  	}
   179  	return t.(testing.TB), true
   180  }
   181  
   182  // initScriptDirs creates the initial directory structure in s for unpacking a
   183  // cmd/go script.
   184  func initScriptDirs(t testing.TB, s *script.State) (telemetryDir string) {
   185  	must := func(err error) {
   186  		if err != nil {
   187  			t.Helper()
   188  			t.Fatal(err)
   189  		}
   190  	}
   191  
   192  	work := s.Getwd()
   193  	must(s.Setenv("WORK", work))
   194  
   195  	telemetryDir = filepath.Join(work, "telemetry")
   196  	must(os.MkdirAll(telemetryDir, 0777))
   197  	must(s.Setenv("TEST_TELEMETRY_DIR", filepath.Join(work, "telemetry")))
   198  
   199  	must(os.MkdirAll(filepath.Join(work, "tmp"), 0777))
   200  	must(s.Setenv(tempEnvName(), filepath.Join(work, "tmp")))
   201  
   202  	gopath := filepath.Join(work, "gopath")
   203  	must(s.Setenv("GOPATH", gopath))
   204  	gopathSrc := filepath.Join(gopath, "src")
   205  	must(os.MkdirAll(gopathSrc, 0777))
   206  	must(s.Chdir(gopathSrc))
   207  	return telemetryDir
   208  }
   209  
   210  func scriptEnv(srv *vcstest.Server, srvCertFile string) ([]string, error) {
   211  	httpURL, err := url.Parse(srv.HTTP.URL)
   212  	if err != nil {
   213  		return nil, err
   214  	}
   215  	httpsURL, err := url.Parse(srv.HTTPS.URL)
   216  	if err != nil {
   217  		return nil, err
   218  	}
   219  	env := []string{
   220  		pathEnvName() + "=" + testBin + string(filepath.ListSeparator) + os.Getenv(pathEnvName()),
   221  		homeEnvName() + "=/no-home",
   222  		"CCACHE_DISABLE=1", // ccache breaks with non-existent HOME
   223  		"GOARCH=" + runtime.GOARCH,
   224  		"TESTGO_GOHOSTARCH=" + goHostArch,
   225  		"GOCACHE=" + testGOCACHE,
   226  		"GOCOVERDIR=" + os.Getenv("GOCOVERDIR"),
   227  		"GODEBUG=" + os.Getenv("GODEBUG"),
   228  		"GOEXE=" + cfg.ExeSuffix,
   229  		"GOEXPERIMENT=" + os.Getenv("GOEXPERIMENT"),
   230  		"GOOS=" + runtime.GOOS,
   231  		"TESTGO_GOHOSTOS=" + goHostOS,
   232  		"GOPROXY=" + proxyURL,
   233  		"GOPRIVATE=",
   234  		"GOROOT=" + testGOROOT,
   235  		"GOTRACEBACK=system",
   236  		"TESTGONETWORK=panic", // allow only local connections by default; the [net] condition resets this
   237  		"TESTGO_GOROOT=" + testGOROOT,
   238  		"TESTGO_EXE=" + testGo,
   239  		"TESTGO_VCSTEST_HOST=" + httpURL.Host,
   240  		"TESTGO_VCSTEST_TLS_HOST=" + httpsURL.Host,
   241  		"TESTGO_VCSTEST_CERT=" + srvCertFile,
   242  		"TESTGONETWORK=panic", // cleared by the [net] condition
   243  		"GOSUMDB=" + testSumDBVerifierKey,
   244  		"GONOPROXY=",
   245  		"GONOSUMDB=",
   246  		"GOVCS=*:all",
   247  		"devnull=" + os.DevNull,
   248  		"goversion=" + gover.Local(),
   249  		"CMDGO_TEST_RUN_MAIN=true",
   250  		"HGRCPATH=",
   251  		"GOTOOLCHAIN=auto",
   252  		"newline=\n",
   253  	}
   254  
   255  	if testenv.Builder() != "" || os.Getenv("GIT_TRACE_CURL") == "1" {
   256  		// To help diagnose https://go.dev/issue/52545,
   257  		// enable tracing for Git HTTPS requests.
   258  		env = append(env,
   259  			"GIT_TRACE_CURL=1",
   260  			"GIT_TRACE_CURL_NO_DATA=1",
   261  			"GIT_REDACT_COOKIES=o,SSO,GSSO_Uberproxy")
   262  	}
   263  	if testing.Short() {
   264  		// VCS commands are always somewhat slow: they either require access to external hosts,
   265  		// or they require our intercepted vcs-test.golang.org to regenerate the repository.
   266  		// Require all tests that use VCS commands to be skipped in short mode.
   267  		env = append(env, "TESTGOVCS=panic")
   268  	}
   269  
   270  	if os.Getenv("CGO_ENABLED") != "" || runtime.GOOS != goHostOS || runtime.GOARCH != goHostArch {
   271  		// If the actual CGO_ENABLED might not match the cmd/go default, set it
   272  		// explicitly in the environment. Otherwise, leave it unset so that we also
   273  		// cover the default behaviors.
   274  		env = append(env, "CGO_ENABLED="+cgoEnabled)
   275  	}
   276  
   277  	for _, key := range extraEnvKeys {
   278  		if val, ok := os.LookupEnv(key); ok {
   279  			env = append(env, key+"="+val)
   280  		}
   281  	}
   282  
   283  	return env, nil
   284  }
   285  
   286  var extraEnvKeys = []string{
   287  	"SYSTEMROOT",         // must be preserved on Windows to find DLLs; golang.org/issue/25210
   288  	"WINDIR",             // must be preserved on Windows to be able to run PowerShell command; golang.org/issue/30711
   289  	"LD_LIBRARY_PATH",    // must be preserved on Unix systems to find shared libraries
   290  	"LIBRARY_PATH",       // allow override of non-standard static library paths
   291  	"C_INCLUDE_PATH",     // allow override non-standard include paths
   292  	"CC",                 // don't lose user settings when invoking cgo
   293  	"GO_TESTING_GOTOOLS", // for gccgo testing
   294  	"GCCGO",              // for gccgo testing
   295  	"GCCGOTOOLDIR",       // for gccgo testing
   296  }
   297  
   298  // updateSum runs 'go mod tidy', 'go list -mod=mod -m all', or
   299  // 'go list -mod=mod all' in the test's current directory if a file named
   300  // "go.mod" is present after the archive has been extracted. updateSum modifies
   301  // archive and returns true if go.mod or go.sum were changed.
   302  func updateSum(t testing.TB, e *script.Engine, s *script.State, archive *txtar.Archive) (rewrite bool) {
   303  	gomodIdx, gosumIdx := -1, -1
   304  	for i := range archive.Files {
   305  		switch archive.Files[i].Name {
   306  		case "go.mod":
   307  			gomodIdx = i
   308  		case "go.sum":
   309  			gosumIdx = i
   310  		}
   311  	}
   312  	if gomodIdx < 0 {
   313  		return false
   314  	}
   315  
   316  	var cmd string
   317  	switch *testSum {
   318  	case "tidy":
   319  		cmd = "go mod tidy"
   320  	case "listm":
   321  		cmd = "go list -m -mod=mod all"
   322  	case "listall":
   323  		cmd = "go list -mod=mod all"
   324  	default:
   325  		t.Fatalf(`unknown value for -testsum %q; may be "tidy", "listm", or "listall"`, *testSum)
   326  	}
   327  
   328  	log := new(strings.Builder)
   329  	err := e.Execute(s, "updateSum", bufio.NewReader(strings.NewReader(cmd)), log)
   330  	if log.Len() > 0 {
   331  		t.Logf("%s", log)
   332  	}
   333  	if err != nil {
   334  		t.Fatal(err)
   335  	}
   336  
   337  	newGomodData, err := os.ReadFile(s.Path("go.mod"))
   338  	if err != nil {
   339  		t.Fatalf("reading go.mod after -testsum: %v", err)
   340  	}
   341  	if !bytes.Equal(newGomodData, archive.Files[gomodIdx].Data) {
   342  		archive.Files[gomodIdx].Data = newGomodData
   343  		rewrite = true
   344  	}
   345  
   346  	newGosumData, err := os.ReadFile(s.Path("go.sum"))
   347  	if err != nil && !os.IsNotExist(err) {
   348  		t.Fatalf("reading go.sum after -testsum: %v", err)
   349  	}
   350  	switch {
   351  	case os.IsNotExist(err) && gosumIdx >= 0:
   352  		// go.sum was deleted.
   353  		rewrite = true
   354  		archive.Files = append(archive.Files[:gosumIdx], archive.Files[gosumIdx+1:]...)
   355  	case err == nil && gosumIdx < 0:
   356  		// go.sum was created.
   357  		rewrite = true
   358  		gosumIdx = gomodIdx + 1
   359  		archive.Files = append(archive.Files, txtar.File{})
   360  		copy(archive.Files[gosumIdx+1:], archive.Files[gosumIdx:])
   361  		archive.Files[gosumIdx] = txtar.File{Name: "go.sum", Data: newGosumData}
   362  	case err == nil && gosumIdx >= 0 && !bytes.Equal(newGosumData, archive.Files[gosumIdx].Data):
   363  		// go.sum was changed.
   364  		rewrite = true
   365  		archive.Files[gosumIdx].Data = newGosumData
   366  	}
   367  	return rewrite
   368  }
   369  
   370  func readCounters(t *testing.T, telemetryDir string) map[string]uint64 {
   371  	localDir := filepath.Join(telemetryDir, "local")
   372  	dirents, err := os.ReadDir(localDir)
   373  	if err != nil {
   374  		if os.IsNotExist(err) {
   375  			return nil // The Go command didn't ever run so the local dir wasn't created
   376  		}
   377  		t.Fatalf("reading telemetry local dir: %v", err)
   378  	}
   379  	totals := map[string]uint64{}
   380  	for _, dirent := range dirents {
   381  		if dirent.IsDir() || !strings.HasSuffix(dirent.Name(), ".count") {
   382  			// not a counter file
   383  			continue
   384  		}
   385  		counters, _, err := countertest.ReadFile(filepath.Join(localDir, dirent.Name()))
   386  		if err != nil {
   387  			t.Fatalf("reading counter file: %v", err)
   388  		}
   389  		for k, v := range counters {
   390  			totals[k] += v
   391  		}
   392  	}
   393  
   394  	return totals
   395  }
   396  
   397  func checkCounters(t *testing.T, telemetryDir string) {
   398  	counters := readCounters(t, telemetryDir)
   399  	if _, ok := scriptGoInvoked.Load(testing.TB(t)); ok {
   400  		if !disabledOnPlatform && len(counters) == 0 {
   401  			t.Fatal("go was invoked but no counters were incremented")
   402  		}
   403  	}
   404  }
   405  
   406  // Copied from https://go.googlesource.com/telemetry/+/5f08a0cbff3f/internal/telemetry/mode.go#122
   407  // TODO(go.dev/issues/66205): replace this with the public API once it becomes available.
   408  //
   409  // disabledOnPlatform indicates whether telemetry is disabled
   410  // due to bugs in the current platform.
   411  const disabledOnPlatform = false ||
   412  	// The following platforms could potentially be supported in the future:
   413  	runtime.GOOS == "openbsd" || // #60614
   414  	runtime.GOOS == "solaris" || // #60968 #60970
   415  	runtime.GOOS == "android" || // #60967
   416  	runtime.GOOS == "illumos" || // #65544
   417  	// These platforms fundamentally can't be supported:
   418  	runtime.GOOS == "js" || // #60971
   419  	runtime.GOOS == "wasip1" || // #60971
   420  	runtime.GOOS == "plan9" // https://github.com/golang/go/issues/57540#issuecomment-1470766639
   421
View as plain text