tests.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package tests
     6  
     7  import (
     8  	_ "embed"
     9  	"fmt"
    10  	"go/ast"
    11  	"go/token"
    12  	"go/types"
    13  	"regexp"
    14  	"strings"
    15  	"unicode"
    16  	"unicode/utf8"
    17  
    18  	"golang.org/x/tools/go/analysis"
    19  	"golang.org/x/tools/go/analysis/passes/internal/analysisutil"
    20  )
    21  
    22  //go:embed doc.go
    23  var doc string
    24  
    25  var Analyzer = &analysis.Analyzer{
    26  	Name: "tests",
    27  	Doc:  analysisutil.MustExtractDoc(doc, "tests"),
    28  	URL:  "https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/tests",
    29  	Run:  run,
    30  }
    31  
    32  var acceptedFuzzTypes = []types.Type{
    33  	types.Typ[types.String],
    34  	types.Typ[types.Bool],
    35  	types.Typ[types.Float32],
    36  	types.Typ[types.Float64],
    37  	types.Typ[types.Int],
    38  	types.Typ[types.Int8],
    39  	types.Typ[types.Int16],
    40  	types.Typ[types.Int32],
    41  	types.Typ[types.Int64],
    42  	types.Typ[types.Uint],
    43  	types.Typ[types.Uint8],
    44  	types.Typ[types.Uint16],
    45  	types.Typ[types.Uint32],
    46  	types.Typ[types.Uint64],
    47  	types.NewSlice(types.Universe.Lookup("byte").Type()),
    48  }
    49  
    50  func run(pass *analysis.Pass) (interface{}, error) {
    51  	for _, f := range pass.Files {
    52  		if !strings.HasSuffix(pass.Fset.File(f.Pos()).Name(), "_test.go") {
    53  			continue
    54  		}
    55  		for _, decl := range f.Decls {
    56  			fn, ok := decl.(*ast.FuncDecl)
    57  			if !ok || fn.Recv != nil {
    58  				// Ignore non-functions or functions with receivers.
    59  				continue
    60  			}
    61  			switch {
    62  			case strings.HasPrefix(fn.Name.Name, "Example"):
    63  				checkExampleName(pass, fn)
    64  				checkExampleOutput(pass, fn, f.Comments)
    65  			case strings.HasPrefix(fn.Name.Name, "Test"):
    66  				checkTest(pass, fn, "Test")
    67  			case strings.HasPrefix(fn.Name.Name, "Benchmark"):
    68  				checkTest(pass, fn, "Benchmark")
    69  			case strings.HasPrefix(fn.Name.Name, "Fuzz"):
    70  				checkTest(pass, fn, "Fuzz")
    71  				checkFuzz(pass, fn)
    72  			}
    73  		}
    74  	}
    75  	return nil, nil
    76  }
    77  
    78  // checkFuzz checks the contents of a fuzz function.
    79  func checkFuzz(pass *analysis.Pass, fn *ast.FuncDecl) {
    80  	params := checkFuzzCall(pass, fn)
    81  	if params != nil {
    82  		checkAddCalls(pass, fn, params)
    83  	}
    84  }
    85  
    86  // checkFuzzCall checks the arguments of f.Fuzz() calls:
    87  //
    88  //  1. f.Fuzz() should call a function and it should be of type (*testing.F).Fuzz().
    89  //  2. The called function in f.Fuzz(func(){}) should not return result.
    90  //  3. First argument of func() should be of type *testing.T
    91  //  4. Second argument onwards should be of type []byte, string, bool, byte,
    92  //     rune, float32, float64, int, int8, int16, int32, int64, uint, uint8, uint16,
    93  //     uint32, uint64
    94  //  5. func() must not call any *F methods, e.g. (*F).Log, (*F).Error, (*F).Skip
    95  //     The only *F methods that are allowed in the (*F).Fuzz function are (*F).Failed and (*F).Name.
    96  //
    97  // Returns the list of parameters to the fuzz function, if they are valid fuzz parameters.
    98  func checkFuzzCall(pass *analysis.Pass, fn *ast.FuncDecl) (params *types.Tuple) {
    99  	ast.Inspect(fn, func(n ast.Node) bool {
   100  		call, ok := n.(*ast.CallExpr)
   101  		if ok {
   102  			if !isFuzzTargetDotFuzz(pass, call) {
   103  				return true
   104  			}
   105  
   106  			// Only one argument (func) must be passed to (*testing.F).Fuzz.
   107  			if len(call.Args) != 1 {
   108  				return true
   109  			}
   110  			expr := call.Args[0]
   111  			if pass.TypesInfo.Types[expr].Type == nil {
   112  				return true
   113  			}
   114  			t := pass.TypesInfo.Types[expr].Type.Underlying()
   115  			tSign, argOk := t.(*types.Signature)
   116  			// Argument should be a function
   117  			if !argOk {
   118  				pass.ReportRangef(expr, "argument to Fuzz must be a function")
   119  				return false
   120  			}
   121  			// ff Argument function should not return
   122  			if tSign.Results().Len() != 0 {
   123  				pass.ReportRangef(expr, "fuzz target must not return any value")
   124  			}
   125  			// ff Argument function should have 1 or more argument
   126  			if tSign.Params().Len() == 0 {
   127  				pass.ReportRangef(expr, "fuzz target must have 1 or more argument")
   128  				return false
   129  			}
   130  			ok := validateFuzzArgs(pass, tSign.Params(), expr)
   131  			if ok && params == nil {
   132  				params = tSign.Params()
   133  			}
   134  			// Inspect the function that was passed as an argument to make sure that
   135  			// there are no calls to *F methods, except for Name and Failed.
   136  			ast.Inspect(expr, func(n ast.Node) bool {
   137  				if call, ok := n.(*ast.CallExpr); ok {
   138  					if !isFuzzTargetDot(pass, call, "") {
   139  						return true
   140  					}
   141  					if !isFuzzTargetDot(pass, call, "Name") && !isFuzzTargetDot(pass, call, "Failed") {
   142  						pass.ReportRangef(call, "fuzz target must not call any *F methods")
   143  					}
   144  				}
   145  				return true
   146  			})
   147  			// We do not need to look at any calls to f.Fuzz inside of a Fuzz call,
   148  			// since they are not allowed.
   149  			return false
   150  		}
   151  		return true
   152  	})
   153  	return params
   154  }
   155  
   156  // checkAddCalls checks that the arguments of f.Add calls have the same number and type of arguments as
   157  // the signature of the function passed to (*testing.F).Fuzz
   158  func checkAddCalls(pass *analysis.Pass, fn *ast.FuncDecl, params *types.Tuple) {
   159  	ast.Inspect(fn, func(n ast.Node) bool {
   160  		call, ok := n.(*ast.CallExpr)
   161  		if ok {
   162  			if !isFuzzTargetDotAdd(pass, call) {
   163  				return true
   164  			}
   165  
   166  			// The first argument to function passed to (*testing.F).Fuzz is (*testing.T).
   167  			if len(call.Args) != params.Len()-1 {
   168  				pass.ReportRangef(call, "wrong number of values in call to (*testing.F).Add: %d, fuzz target expects %d", len(call.Args), params.Len()-1)
   169  				return true
   170  			}
   171  			var mismatched []int
   172  			for i, expr := range call.Args {
   173  				if pass.TypesInfo.Types[expr].Type == nil {
   174  					return true
   175  				}
   176  				t := pass.TypesInfo.Types[expr].Type
   177  				if !types.Identical(t, params.At(i+1).Type()) {
   178  					mismatched = append(mismatched, i)
   179  				}
   180  			}
   181  			// If just one of the types is mismatched report for that
   182  			// type only. Otherwise report for the whole call to (*testing.F).Add
   183  			if len(mismatched) == 1 {
   184  				i := mismatched[0]
   185  				expr := call.Args[i]
   186  				t := pass.TypesInfo.Types[expr].Type
   187  				pass.ReportRangef(expr, fmt.Sprintf("mismatched type in call to (*testing.F).Add: %v, fuzz target expects %v", t, params.At(i+1).Type()))
   188  			} else if len(mismatched) > 1 {
   189  				var gotArgs, wantArgs []types.Type
   190  				for i := 0; i < len(call.Args); i++ {
   191  					gotArgs, wantArgs = append(gotArgs, pass.TypesInfo.Types[call.Args[i]].Type), append(wantArgs, params.At(i+1).Type())
   192  				}
   193  				pass.ReportRangef(call, fmt.Sprintf("mismatched types in call to (*testing.F).Add: %v, fuzz target expects %v", gotArgs, wantArgs))
   194  			}
   195  		}
   196  		return true
   197  	})
   198  }
   199  
   200  // isFuzzTargetDotFuzz reports whether call is (*testing.F).Fuzz().
   201  func isFuzzTargetDotFuzz(pass *analysis.Pass, call *ast.CallExpr) bool {
   202  	return isFuzzTargetDot(pass, call, "Fuzz")
   203  }
   204  
   205  // isFuzzTargetDotAdd reports whether call is (*testing.F).Add().
   206  func isFuzzTargetDotAdd(pass *analysis.Pass, call *ast.CallExpr) bool {
   207  	return isFuzzTargetDot(pass, call, "Add")
   208  }
   209  
   210  // isFuzzTargetDot reports whether call is (*testing.F).<name>().
   211  func isFuzzTargetDot(pass *analysis.Pass, call *ast.CallExpr, name string) bool {
   212  	if selExpr, ok := call.Fun.(*ast.SelectorExpr); ok {
   213  		if !isTestingType(pass.TypesInfo.Types[selExpr.X].Type, "F") {
   214  			return false
   215  		}
   216  		if name == "" || selExpr.Sel.Name == name {
   217  			return true
   218  		}
   219  	}
   220  	return false
   221  }
   222  
   223  // Validate the arguments of fuzz target.
   224  func validateFuzzArgs(pass *analysis.Pass, params *types.Tuple, expr ast.Expr) bool {
   225  	fLit, isFuncLit := expr.(*ast.FuncLit)
   226  	exprRange := expr
   227  	ok := true
   228  	if !isTestingType(params.At(0).Type(), "T") {
   229  		if isFuncLit {
   230  			exprRange = fLit.Type.Params.List[0].Type
   231  		}
   232  		pass.ReportRangef(exprRange, "the first parameter of a fuzz target must be *testing.T")
   233  		ok = false
   234  	}
   235  	for i := 1; i < params.Len(); i++ {
   236  		if !isAcceptedFuzzType(params.At(i).Type()) {
   237  			if isFuncLit {
   238  				curr := 0
   239  				for _, field := range fLit.Type.Params.List {
   240  					curr += len(field.Names)
   241  					if i < curr {
   242  						exprRange = field.Type
   243  						break
   244  					}
   245  				}
   246  			}
   247  			pass.ReportRangef(exprRange, "fuzzing arguments can only have the following types: "+formatAcceptedFuzzType())
   248  			ok = false
   249  		}
   250  	}
   251  	return ok
   252  }
   253  
   254  func isTestingType(typ types.Type, testingType string) bool {
   255  	// No Unalias here: I doubt "go test" recognizes
   256  	// "type A = *testing.T; func Test(A) {}" as a test.
   257  	ptr, ok := typ.(*types.Pointer)
   258  	if !ok {
   259  		return false
   260  	}
   261  	return analysisutil.IsNamedType(ptr.Elem(), "testing", testingType)
   262  }
   263  
   264  // Validate that fuzz target function's arguments are of accepted types.
   265  func isAcceptedFuzzType(paramType types.Type) bool {
   266  	for _, typ := range acceptedFuzzTypes {
   267  		if types.Identical(typ, paramType) {
   268  			return true
   269  		}
   270  	}
   271  	return false
   272  }
   273  
   274  func formatAcceptedFuzzType() string {
   275  	var acceptedFuzzTypesStrings []string
   276  	for _, typ := range acceptedFuzzTypes {
   277  		acceptedFuzzTypesStrings = append(acceptedFuzzTypesStrings, typ.String())
   278  	}
   279  	acceptedFuzzTypesMsg := strings.Join(acceptedFuzzTypesStrings, ", ")
   280  	return acceptedFuzzTypesMsg
   281  }
   282  
   283  func isExampleSuffix(s string) bool {
   284  	r, size := utf8.DecodeRuneInString(s)
   285  	return size > 0 && unicode.IsLower(r)
   286  }
   287  
   288  func isTestSuffix(name string) bool {
   289  	if len(name) == 0 {
   290  		// "Test" is ok.
   291  		return true
   292  	}
   293  	r, _ := utf8.DecodeRuneInString(name)
   294  	return !unicode.IsLower(r)
   295  }
   296  
   297  func isTestParam(typ ast.Expr, wantType string) bool {
   298  	ptr, ok := typ.(*ast.StarExpr)
   299  	if !ok {
   300  		// Not a pointer.
   301  		return false
   302  	}
   303  	// No easy way of making sure it's a *testing.T or *testing.B:
   304  	// ensure the name of the type matches.
   305  	if name, ok := ptr.X.(*ast.Ident); ok {
   306  		return name.Name == wantType
   307  	}
   308  	if sel, ok := ptr.X.(*ast.SelectorExpr); ok {
   309  		return sel.Sel.Name == wantType
   310  	}
   311  	return false
   312  }
   313  
   314  func lookup(pkg *types.Package, name string) []types.Object {
   315  	if o := pkg.Scope().Lookup(name); o != nil {
   316  		return []types.Object{o}
   317  	}
   318  
   319  	var ret []types.Object
   320  	// Search through the imports to see if any of them define name.
   321  	// It's hard to tell in general which package is being tested, so
   322  	// for the purposes of the analysis, allow the object to appear
   323  	// in any of the imports. This guarantees there are no false positives
   324  	// because the example needs to use the object so it must be defined
   325  	// in the package or one if its imports. On the other hand, false
   326  	// negatives are possible, but should be rare.
   327  	for _, imp := range pkg.Imports() {
   328  		if obj := imp.Scope().Lookup(name); obj != nil {
   329  			ret = append(ret, obj)
   330  		}
   331  	}
   332  	return ret
   333  }
   334  
   335  // This pattern is taken from /go/src/go/doc/example.go
   336  var outputRe = regexp.MustCompile(`(?i)^[[:space:]]*(unordered )?output:`)
   337  
   338  type commentMetadata struct {
   339  	isOutput bool
   340  	pos      token.Pos
   341  }
   342  
   343  func checkExampleOutput(pass *analysis.Pass, fn *ast.FuncDecl, fileComments []*ast.CommentGroup) {
   344  	commentsInExample := []commentMetadata{}
   345  	numOutputs := 0
   346  
   347  	// Find the comment blocks that are in the example. These comments are
   348  	// guaranteed to be in order of appearance.
   349  	for _, cg := range fileComments {
   350  		if cg.Pos() < fn.Pos() {
   351  			continue
   352  		} else if cg.End() > fn.End() {
   353  			break
   354  		}
   355  
   356  		isOutput := outputRe.MatchString(cg.Text())
   357  		if isOutput {
   358  			numOutputs++
   359  		}
   360  
   361  		commentsInExample = append(commentsInExample, commentMetadata{
   362  			isOutput: isOutput,
   363  			pos:      cg.Pos(),
   364  		})
   365  	}
   366  
   367  	// Change message based on whether there are multiple output comment blocks.
   368  	msg := "output comment block must be the last comment block"
   369  	if numOutputs > 1 {
   370  		msg = "there can only be one output comment block per example"
   371  	}
   372  
   373  	for i, cg := range commentsInExample {
   374  		// Check for output comments that are not the last comment in the example.
   375  		isLast := (i == len(commentsInExample)-1)
   376  		if cg.isOutput && !isLast {
   377  			pass.Report(
   378  				analysis.Diagnostic{
   379  					Pos:     cg.pos,
   380  					Message: msg,
   381  				},
   382  			)
   383  		}
   384  	}
   385  }
   386  
   387  func checkExampleName(pass *analysis.Pass, fn *ast.FuncDecl) {
   388  	fnName := fn.Name.Name
   389  	if params := fn.Type.Params; len(params.List) != 0 {
   390  		pass.Reportf(fn.Pos(), "%s should be niladic", fnName)
   391  	}
   392  	if results := fn.Type.Results; results != nil && len(results.List) != 0 {
   393  		pass.Reportf(fn.Pos(), "%s should return nothing", fnName)
   394  	}
   395  	if tparams := fn.Type.TypeParams; tparams != nil && len(tparams.List) > 0 {
   396  		pass.Reportf(fn.Pos(), "%s should not have type params", fnName)
   397  	}
   398  
   399  	if fnName == "Example" {
   400  		// Nothing more to do.
   401  		return
   402  	}
   403  
   404  	var (
   405  		exName = strings.TrimPrefix(fnName, "Example")
   406  		elems  = strings.SplitN(exName, "_", 3)
   407  		ident  = elems[0]
   408  		objs   = lookup(pass.Pkg, ident)
   409  	)
   410  	if ident != "" && len(objs) == 0 {
   411  		// Check ExampleFoo and ExampleBadFoo.
   412  		pass.Reportf(fn.Pos(), "%s refers to unknown identifier: %s", fnName, ident)
   413  		// Abort since obj is absent and no subsequent checks can be performed.
   414  		return
   415  	}
   416  	if len(elems) < 2 {
   417  		// Nothing more to do.
   418  		return
   419  	}
   420  
   421  	if ident == "" {
   422  		// Check Example_suffix and Example_BadSuffix.
   423  		if residual := strings.TrimPrefix(exName, "_"); !isExampleSuffix(residual) {
   424  			pass.Reportf(fn.Pos(), "%s has malformed example suffix: %s", fnName, residual)
   425  		}
   426  		return
   427  	}
   428  
   429  	mmbr := elems[1]
   430  	if !isExampleSuffix(mmbr) {
   431  		// Check ExampleFoo_Method and ExampleFoo_BadMethod.
   432  		found := false
   433  		// Check if Foo.Method exists in this package or its imports.
   434  		for _, obj := range objs {
   435  			if obj, _, _ := types.LookupFieldOrMethod(obj.Type(), true, obj.Pkg(), mmbr); obj != nil {
   436  				found = true
   437  				break
   438  			}
   439  		}
   440  		if !found {
   441  			pass.Reportf(fn.Pos(), "%s refers to unknown field or method: %s.%s", fnName, ident, mmbr)
   442  		}
   443  	}
   444  	if len(elems) == 3 && !isExampleSuffix(elems[2]) {
   445  		// Check ExampleFoo_Method_suffix and ExampleFoo_Method_Badsuffix.
   446  		pass.Reportf(fn.Pos(), "%s has malformed example suffix: %s", fnName, elems[2])
   447  	}
   448  }
   449  
   450  type tokenRange struct {
   451  	p, e token.Pos
   452  }
   453  
   454  func (r tokenRange) Pos() token.Pos {
   455  	return r.p
   456  }
   457  
   458  func (r tokenRange) End() token.Pos {
   459  	return r.e
   460  }
   461  
   462  func checkTest(pass *analysis.Pass, fn *ast.FuncDecl, prefix string) {
   463  	// Want functions with 0 results and 1 parameter.
   464  	if fn.Type.Results != nil && len(fn.Type.Results.List) > 0 ||
   465  		fn.Type.Params == nil ||
   466  		len(fn.Type.Params.List) != 1 ||
   467  		len(fn.Type.Params.List[0].Names) > 1 {
   468  		return
   469  	}
   470  
   471  	// The param must look like a *testing.T or *testing.B.
   472  	if !isTestParam(fn.Type.Params.List[0].Type, prefix[:1]) {
   473  		return
   474  	}
   475  
   476  	if tparams := fn.Type.TypeParams; tparams != nil && len(tparams.List) > 0 {
   477  		// Note: cmd/go/internal/load also errors about TestXXX and BenchmarkXXX functions with type parameters.
   478  		// We have currently decided to also warn before compilation/package loading. This can help users in IDEs.
   479  		at := tokenRange{tparams.Opening, tparams.Closing}
   480  		pass.ReportRangef(at, "%s has type parameters: it will not be run by go test as a %sXXX function", fn.Name.Name, prefix)
   481  	}
   482  
   483  	if !isTestSuffix(fn.Name.Name[len(prefix):]) {
   484  		pass.ReportRangef(fn.Name, "%s has malformed name: first letter after '%s' must not be lowercase", fn.Name.Name, prefix)
   485  	}
   486  }
   487
View as plain text