buildtool.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Build toolchain using Go bootstrap version.
     6  //
     7  // The general strategy is to copy the source files we need into
     8  // a new GOPATH workspace, adjust import paths appropriately,
     9  // invoke the Go bootstrap toolchains go command to build those sources,
    10  // and then copy the binaries back.
    11  
    12  package main
    13  
    14  import (
    15  	"fmt"
    16  	"os"
    17  	"path/filepath"
    18  	"regexp"
    19  	"strings"
    20  )
    21  
    22  // bootstrapDirs is a list of directories holding code that must be
    23  // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
    24  // All directories in this list are relative to and must be below $GOROOT/src.
    25  //
    26  // The list has two kinds of entries: names beginning with cmd/ with
    27  // no other slashes, which are commands, and other paths, which are packages
    28  // supporting the commands. Packages in the standard library can be listed
    29  // if a newer copy needs to be substituted for the Go bootstrap copy when used
    30  // by the command packages. Paths ending with /... automatically
    31  // include all packages within subdirectories as well.
    32  // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
    33  var bootstrapDirs = []string{
    34  	"cmp",
    35  	"cmd/asm",
    36  	"cmd/asm/internal/...",
    37  	"cmd/cgo",
    38  	"cmd/compile",
    39  	"cmd/compile/internal/...",
    40  	"cmd/internal/archive",
    41  	"cmd/internal/bio",
    42  	"cmd/internal/codesign",
    43  	"cmd/internal/dwarf",
    44  	"cmd/internal/edit",
    45  	"cmd/internal/gcprog",
    46  	"cmd/internal/goobj",
    47  	"cmd/internal/notsha256",
    48  	"cmd/internal/obj/...",
    49  	"cmd/internal/objabi",
    50  	"cmd/internal/pgo",
    51  	"cmd/internal/pkgpath",
    52  	"cmd/internal/quoted",
    53  	"cmd/internal/src",
    54  	"cmd/internal/sys",
    55  	"cmd/internal/telemetry",
    56  	"cmd/internal/telemetry/counter",
    57  	"cmd/link",
    58  	"cmd/link/internal/...",
    59  	"compress/flate",
    60  	"compress/zlib",
    61  	"container/heap",
    62  	"debug/dwarf",
    63  	"debug/elf",
    64  	"debug/macho",
    65  	"debug/pe",
    66  	"go/build/constraint",
    67  	"go/constant",
    68  	"go/version",
    69  	"internal/abi",
    70  	"internal/coverage",
    71  	"cmd/internal/cov/covcmd",
    72  	"internal/bisect",
    73  	"internal/buildcfg",
    74  	"internal/goarch",
    75  	"internal/godebugs",
    76  	"internal/goexperiment",
    77  	"internal/goroot",
    78  	"internal/gover",
    79  	"internal/goversion",
    80  	// internal/lazyregexp is provided by Go 1.17, which permits it to
    81  	// be imported by other packages in this list, but is not provided
    82  	// by the Go 1.17 version of gccgo. It's on this list only to
    83  	// support gccgo, and can be removed if we require gccgo 14 or later.
    84  	"internal/lazyregexp",
    85  	"internal/pkgbits",
    86  	"internal/platform",
    87  	"internal/profile",
    88  	"internal/race",
    89  	"internal/saferio",
    90  	"internal/syscall/unix",
    91  	"internal/types/errors",
    92  	"internal/unsafeheader",
    93  	"internal/xcoff",
    94  	"internal/zstd",
    95  	"math/bits",
    96  	"sort",
    97  }
    98  
    99  // File prefixes that are ignored by go/build anyway, and cause
   100  // problems with editor generated temporary files (#18931).
   101  var ignorePrefixes = []string{
   102  	".",
   103  	"_",
   104  	"#",
   105  }
   106  
   107  // File suffixes that use build tags introduced since Go 1.17.
   108  // These must not be copied into the bootstrap build directory.
   109  // Also ignore test files.
   110  var ignoreSuffixes = []string{
   111  	"_test.s",
   112  	"_test.go",
   113  	// Skip PGO profile. No need to build toolchain1 compiler
   114  	// with PGO. And as it is not a text file the import path
   115  	// rewrite will break it.
   116  	".pgo",
   117  	// Skip editor backup files.
   118  	"~",
   119  }
   120  
   121  var tryDirs = []string{
   122  	"sdk/go1.17",
   123  	"go1.17",
   124  }
   125  
   126  func bootstrapBuildTools() {
   127  	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
   128  	if goroot_bootstrap == "" {
   129  		home := os.Getenv("HOME")
   130  		goroot_bootstrap = pathf("%s/go1.4", home)
   131  		for _, d := range tryDirs {
   132  			if p := pathf("%s/%s", home, d); isdir(p) {
   133  				goroot_bootstrap = p
   134  			}
   135  		}
   136  	}
   137  	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
   138  
   139  	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
   140  	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
   141  
   142  	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
   143  	// We use a subdirectory of $GOROOT/pkg because that's the
   144  	// space within $GOROOT where we store all generated objects.
   145  	// We could use a temporary directory outside $GOROOT instead,
   146  	// but it is easier to debug on failure if the files are in a known location.
   147  	workspace := pathf("%s/pkg/bootstrap", goroot)
   148  	xremoveall(workspace)
   149  	xatexit(func() { xremoveall(workspace) })
   150  	base := pathf("%s/src/bootstrap", workspace)
   151  	xmkdirall(base)
   152  
   153  	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
   154  	writefile("module bootstrap\ngo 1.20\n", pathf("%s/%s", base, "go.mod"), 0)
   155  	for _, dir := range bootstrapDirs {
   156  		recurse := strings.HasSuffix(dir, "/...")
   157  		dir = strings.TrimSuffix(dir, "/...")
   158  		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   159  			if err != nil {
   160  				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
   161  			}
   162  
   163  			name := filepath.Base(path)
   164  			src := pathf("%s/src/%s", goroot, path)
   165  			dst := pathf("%s/%s", base, path)
   166  
   167  			if info.IsDir() {
   168  				if !recurse && path != dir || name == "testdata" {
   169  					return filepath.SkipDir
   170  				}
   171  
   172  				xmkdirall(dst)
   173  				if path == "cmd/cgo" {
   174  					// Write to src because we need the file both for bootstrap
   175  					// and for later in the main build.
   176  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
   177  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
   178  				}
   179  				return nil
   180  			}
   181  
   182  			for _, pre := range ignorePrefixes {
   183  				if strings.HasPrefix(name, pre) {
   184  					return nil
   185  				}
   186  			}
   187  			for _, suf := range ignoreSuffixes {
   188  				if strings.HasSuffix(name, suf) {
   189  					return nil
   190  				}
   191  			}
   192  
   193  			text := bootstrapRewriteFile(src)
   194  			writefile(text, dst, 0)
   195  			return nil
   196  		})
   197  	}
   198  
   199  	// Set up environment for invoking Go bootstrap toolchains go command.
   200  	// GOROOT points at Go bootstrap GOROOT,
   201  	// GOPATH points at our bootstrap workspace,
   202  	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
   203  	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
   204  	// so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
   205  	// Restore GOROOT, GOPATH, and GOBIN when done.
   206  	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
   207  	// because setup will take care of those when bootstrapBuildTools returns.
   208  
   209  	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
   210  	os.Setenv("GOROOT", goroot_bootstrap)
   211  
   212  	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
   213  	os.Setenv("GOPATH", workspace)
   214  
   215  	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
   216  	os.Setenv("GOBIN", "")
   217  
   218  	os.Setenv("GOOS", "")
   219  	os.Setenv("GOHOSTOS", "")
   220  	os.Setenv("GOARCH", "")
   221  	os.Setenv("GOHOSTARCH", "")
   222  
   223  	// Run Go bootstrap to build binaries.
   224  	// Use the math_big_pure_go build tag to disable the assembly in math/big
   225  	// which may contain unsupported instructions.
   226  	// Use the purego build tag to disable other assembly code,
   227  	// such as in cmd/internal/notsha256.
   228  	cmd := []string{
   229  		pathf("%s/bin/go", goroot_bootstrap),
   230  		"install",
   231  		"-tags=math_big_pure_go compiler_bootstrap purego",
   232  	}
   233  	if vflag > 0 {
   234  		cmd = append(cmd, "-v")
   235  	}
   236  	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
   237  		cmd = append(cmd, "-toolexec="+tool)
   238  	}
   239  	cmd = append(cmd, "bootstrap/cmd/...")
   240  	run(base, ShowOutput|CheckExit, cmd...)
   241  
   242  	// Copy binaries into tool binary directory.
   243  	for _, name := range bootstrapDirs {
   244  		if !strings.HasPrefix(name, "cmd/") {
   245  			continue
   246  		}
   247  		name = name[len("cmd/"):]
   248  		if !strings.Contains(name, "/") {
   249  			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
   250  		}
   251  	}
   252  
   253  	if vflag > 0 {
   254  		xprintf("\n")
   255  	}
   256  }
   257  
   258  var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
   259  
   260  // isUnneededSSARewriteFile reports whether srcFile is a
   261  // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
   262  // architecture that isn't for the given GOARCH.
   263  //
   264  // When unneeded is true archCaps is the rewrite base filename without
   265  // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
   266  func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
   267  	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
   268  		return "", false
   269  	}
   270  	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
   271  	if fileArch == "" {
   272  		return "", false
   273  	}
   274  	b := fileArch[0]
   275  	if b == '_' || ('a' <= b && b <= 'z') {
   276  		return "", false
   277  	}
   278  	archCaps = fileArch
   279  	fileArch = strings.ToLower(fileArch)
   280  	fileArch = strings.TrimSuffix(fileArch, "splitload")
   281  	fileArch = strings.TrimSuffix(fileArch, "latelower")
   282  	if fileArch == goArch {
   283  		return "", false
   284  	}
   285  	if fileArch == strings.TrimSuffix(goArch, "le") {
   286  		return "", false
   287  	}
   288  	return archCaps, true
   289  }
   290  
   291  func bootstrapRewriteFile(srcFile string) string {
   292  	// During bootstrap, generate dummy rewrite files for
   293  	// irrelevant architectures. We only need to build a bootstrap
   294  	// binary that works for the current gohostarch.
   295  	// This saves 6+ seconds of bootstrap.
   296  	if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
   297  		return fmt.Sprintf(`%spackage ssa
   298  
   299  func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
   300  func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
   301  `, generatedHeader, archCaps, archCaps)
   302  	}
   303  
   304  	return bootstrapFixImports(srcFile)
   305  }
   306  
   307  var (
   308  	importRE      = regexp.MustCompile(`\Aimport\s+(\.|[A-Za-z0-9_]+)?\s*"([^"]+)"\s*(//.*)?\n\z`)
   309  	importBlockRE = regexp.MustCompile(`\A\s*(?:(\.|[A-Za-z0-9_]+)?\s*"([^"]+)")?\s*(//.*)?\n\z`)
   310  )
   311  
   312  func bootstrapFixImports(srcFile string) string {
   313  	text := readfile(srcFile)
   314  	if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
   315  		text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
   316  	}
   317  	lines := strings.SplitAfter(text, "\n")
   318  	inBlock := false
   319  	inComment := false
   320  	for i, line := range lines {
   321  		if strings.HasSuffix(line, "*/\n") {
   322  			inComment = false
   323  		}
   324  		if strings.HasSuffix(line, "/*\n") {
   325  			inComment = true
   326  		}
   327  		if inComment {
   328  			continue
   329  		}
   330  		if strings.HasPrefix(line, "import (") {
   331  			inBlock = true
   332  			continue
   333  		}
   334  		if inBlock && strings.HasPrefix(line, ")") {
   335  			inBlock = false
   336  			continue
   337  		}
   338  
   339  		var m []string
   340  		if !inBlock {
   341  			if !strings.HasPrefix(line, "import ") {
   342  				continue
   343  			}
   344  			m = importRE.FindStringSubmatch(line)
   345  			if m == nil {
   346  				fatalf("%s:%d: invalid import declaration: %q", srcFile, i+1, line)
   347  			}
   348  		} else {
   349  			m = importBlockRE.FindStringSubmatch(line)
   350  			if m == nil {
   351  				fatalf("%s:%d: invalid import block line", srcFile, i+1)
   352  			}
   353  			if m[2] == "" {
   354  				continue
   355  			}
   356  		}
   357  
   358  		path := m[2]
   359  		if strings.HasPrefix(path, "cmd/") {
   360  			path = "bootstrap/" + path
   361  		} else {
   362  			for _, dir := range bootstrapDirs {
   363  				if path == dir {
   364  					path = "bootstrap/" + dir
   365  					break
   366  				}
   367  			}
   368  		}
   369  
   370  		// Rewrite use of internal/reflectlite to be plain reflect.
   371  		if path == "internal/reflectlite" {
   372  			lines[i] = strings.ReplaceAll(line, `"reflect"`, `reflectlite "reflect"`)
   373  			continue
   374  		}
   375  
   376  		// Otherwise, reject direct imports of internal packages,
   377  		// since that implies knowledge of internal details that might
   378  		// change from one bootstrap toolchain to the next.
   379  		// There are many internal packages that are listed in
   380  		// bootstrapDirs and made into bootstrap copies based on the
   381  		// current repo's source code. Those are fine; this is catching
   382  		// references to internal packages in the older bootstrap toolchain.
   383  		if strings.HasPrefix(path, "internal/") {
   384  			fatalf("%s:%d: bootstrap-copied source file cannot import %s", srcFile, i+1, path)
   385  		}
   386  		if path != m[2] {
   387  			lines[i] = strings.ReplaceAll(line, `"`+m[2]+`"`, `"`+path+`"`)
   388  		}
   389  	}
   390  
   391  	lines[0] = generatedHeader + "// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
   392  
   393  	return strings.Join(lines, "")
   394  }
   395
View as plain text