parser_test.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package syntax
     6  
     7  import (
     8  	"bytes"
     9  	"flag"
    10  	"fmt"
    11  	"internal/testenv"
    12  	"os"
    13  	"path/filepath"
    14  	"regexp"
    15  	"runtime"
    16  	"strings"
    17  	"sync"
    18  	"testing"
    19  	"time"
    20  )
    21  
    22  var (
    23  	fast   = flag.Bool("fast", false, "parse package files in parallel")
    24  	verify = flag.Bool("verify", false, "verify idempotent printing")
    25  	src_   = flag.String("src", "parser.go", "source file to parse")
    26  	skip   = flag.String("skip", "", "files matching this regular expression are skipped by TestStdLib")
    27  )
    28  
    29  func TestParse(t *testing.T) {
    30  	ParseFile(*src_, func(err error) { t.Error(err) }, nil, 0)
    31  }
    32  
    33  func TestVerify(t *testing.T) {
    34  	ast, err := ParseFile(*src_, func(err error) { t.Error(err) }, nil, 0)
    35  	if err != nil {
    36  		return // error already reported
    37  	}
    38  	verifyPrint(t, *src_, ast)
    39  }
    40  
    41  func TestStdLib(t *testing.T) {
    42  	if testing.Short() {
    43  		t.Skip("skipping test in short mode")
    44  	}
    45  
    46  	var skipRx *regexp.Regexp
    47  	if *skip != "" {
    48  		var err error
    49  		skipRx, err = regexp.Compile(*skip)
    50  		if err != nil {
    51  			t.Fatalf("invalid argument for -skip (%v)", err)
    52  		}
    53  	}
    54  
    55  	var m1 runtime.MemStats
    56  	runtime.ReadMemStats(&m1)
    57  	start := time.Now()
    58  
    59  	type parseResult struct {
    60  		filename string
    61  		lines    uint
    62  	}
    63  
    64  	goroot := testenv.GOROOT(t)
    65  
    66  	results := make(chan parseResult)
    67  	go func() {
    68  		defer close(results)
    69  		for _, dir := range []string{
    70  			filepath.Join(goroot, "src"),
    71  			filepath.Join(goroot, "misc"),
    72  		} {
    73  			if filepath.Base(dir) == "misc" {
    74  				// cmd/distpack deletes GOROOT/misc, so skip that directory if it isn't present.
    75  				// cmd/distpack also requires GOROOT/VERSION to exist, so use that to
    76  				// suppress false-positive skips.
    77  				if _, err := os.Stat(dir); os.IsNotExist(err) {
    78  					if _, err := os.Stat(filepath.Join(testenv.GOROOT(t), "VERSION")); err == nil {
    79  						fmt.Printf("%s not present; skipping\n", dir)
    80  						continue
    81  					}
    82  				}
    83  			}
    84  
    85  			walkDirs(t, dir, func(filename string) {
    86  				if skipRx != nil && skipRx.MatchString(filename) {
    87  					// Always report skipped files since regexp
    88  					// typos can lead to surprising results.
    89  					fmt.Printf("skipping %s\n", filename)
    90  					return
    91  				}
    92  				if debug {
    93  					fmt.Printf("parsing %s\n", filename)
    94  				}
    95  				ast, err := ParseFile(filename, nil, nil, 0)
    96  				if err != nil {
    97  					t.Error(err)
    98  					return
    99  				}
   100  				if *verify {
   101  					verifyPrint(t, filename, ast)
   102  				}
   103  				results <- parseResult{filename, ast.EOF.Line()}
   104  			})
   105  		}
   106  	}()
   107  
   108  	var count, lines uint
   109  	for res := range results {
   110  		count++
   111  		lines += res.lines
   112  		if testing.Verbose() {
   113  			fmt.Printf("%5d  %s (%d lines)\n", count, res.filename, res.lines)
   114  		}
   115  	}
   116  
   117  	dt := time.Since(start)
   118  	var m2 runtime.MemStats
   119  	runtime.ReadMemStats(&m2)
   120  	dm := float64(m2.TotalAlloc-m1.TotalAlloc) / 1e6
   121  
   122  	fmt.Printf("parsed %d lines (%d files) in %v (%d lines/s)\n", lines, count, dt, int64(float64(lines)/dt.Seconds()))
   123  	fmt.Printf("allocated %.3fMb (%.3fMb/s)\n", dm, dm/dt.Seconds())
   124  }
   125  
   126  func walkDirs(t *testing.T, dir string, action func(string)) {
   127  	entries, err := os.ReadDir(dir)
   128  	if err != nil {
   129  		t.Error(err)
   130  		return
   131  	}
   132  
   133  	var files, dirs []string
   134  	for _, entry := range entries {
   135  		if entry.Type().IsRegular() {
   136  			if strings.HasSuffix(entry.Name(), ".go") {
   137  				path := filepath.Join(dir, entry.Name())
   138  				files = append(files, path)
   139  			}
   140  		} else if entry.IsDir() && entry.Name() != "testdata" {
   141  			path := filepath.Join(dir, entry.Name())
   142  			if !strings.HasSuffix(path, string(filepath.Separator)+"test") {
   143  				dirs = append(dirs, path)
   144  			}
   145  		}
   146  	}
   147  
   148  	if *fast {
   149  		var wg sync.WaitGroup
   150  		wg.Add(len(files))
   151  		for _, filename := range files {
   152  			go func(filename string) {
   153  				defer wg.Done()
   154  				action(filename)
   155  			}(filename)
   156  		}
   157  		wg.Wait()
   158  	} else {
   159  		for _, filename := range files {
   160  			action(filename)
   161  		}
   162  	}
   163  
   164  	for _, dir := range dirs {
   165  		walkDirs(t, dir, action)
   166  	}
   167  }
   168  
   169  func verifyPrint(t *testing.T, filename string, ast1 *File) {
   170  	var buf1 bytes.Buffer
   171  	_, err := Fprint(&buf1, ast1, LineForm)
   172  	if err != nil {
   173  		panic(err)
   174  	}
   175  	bytes1 := buf1.Bytes()
   176  
   177  	ast2, err := Parse(NewFileBase(filename), &buf1, nil, nil, 0)
   178  	if err != nil {
   179  		panic(err)
   180  	}
   181  
   182  	var buf2 bytes.Buffer
   183  	_, err = Fprint(&buf2, ast2, LineForm)
   184  	if err != nil {
   185  		panic(err)
   186  	}
   187  	bytes2 := buf2.Bytes()
   188  
   189  	if !bytes.Equal(bytes1, bytes2) {
   190  		fmt.Printf("--- %s ---\n", filename)
   191  		fmt.Printf("%s\n", bytes1)
   192  		fmt.Println()
   193  
   194  		fmt.Printf("--- %s ---\n", filename)
   195  		fmt.Printf("%s\n", bytes2)
   196  		fmt.Println()
   197  
   198  		t.Error("printed syntax trees do not match")
   199  	}
   200  }
   201  
   202  func TestIssue17697(t *testing.T) {
   203  	_, err := Parse(nil, bytes.NewReader(nil), nil, nil, 0) // return with parser error, don't panic
   204  	if err == nil {
   205  		t.Errorf("no error reported")
   206  	}
   207  }
   208  
   209  func TestParseFile(t *testing.T) {
   210  	_, err := ParseFile("", nil, nil, 0)
   211  	if err == nil {
   212  		t.Error("missing io error")
   213  	}
   214  
   215  	var first error
   216  	_, err = ParseFile("", func(err error) {
   217  		if first == nil {
   218  			first = err
   219  		}
   220  	}, nil, 0)
   221  	if err == nil || first == nil {
   222  		t.Error("missing io error")
   223  	}
   224  	if err != first {
   225  		t.Errorf("got %v; want first error %v", err, first)
   226  	}
   227  }
   228  
   229  // Make sure (PosMax + 1) doesn't overflow when converted to default
   230  // type int (when passed as argument to fmt.Sprintf) on 32bit platforms
   231  // (see test cases below).
   232  var tooLarge int = PosMax + 1
   233  
   234  func TestLineDirectives(t *testing.T) {
   235  	// valid line directives lead to a syntax error after them
   236  	const valid = "syntax error: package statement must be first"
   237  	const filename = "directives.go"
   238  
   239  	for _, test := range []struct {
   240  		src, msg  string
   241  		filename  string
   242  		line, col uint // 1-based; 0 means unknown
   243  	}{
   244  		// ignored //line directives
   245  		{"//\n", valid, filename, 2, 1},            // no directive
   246  		{"//line\n", valid, filename, 2, 1},        // missing colon
   247  		{"//line foo\n", valid, filename, 2, 1},    // missing colon
   248  		{"  //line foo:\n", valid, filename, 2, 1}, // not a line start
   249  		{"//  line foo:\n", valid, filename, 2, 1}, // space between // and line
   250  
   251  		// invalid //line directives with one colon
   252  		{"//line :\n", "invalid line number: ", filename, 1, 9},
   253  		{"//line :x\n", "invalid line number: x", filename, 1, 9},
   254  		{"//line foo :\n", "invalid line number: ", filename, 1, 13},
   255  		{"//line foo:x\n", "invalid line number: x", filename, 1, 12},
   256  		{"//line foo:0\n", "invalid line number: 0", filename, 1, 12},
   257  		{"//line foo:1 \n", "invalid line number: 1 ", filename, 1, 12},
   258  		{"//line foo:-12\n", "invalid line number: -12", filename, 1, 12},
   259  		{"//line C:foo:0\n", "invalid line number: 0", filename, 1, 14},
   260  		{fmt.Sprintf("//line foo:%d\n", tooLarge), fmt.Sprintf("invalid line number: %d", tooLarge), filename, 1, 12},
   261  
   262  		// invalid //line directives with two colons
   263  		{"//line ::\n", "invalid line number: ", filename, 1, 10},
   264  		{"//line ::x\n", "invalid line number: x", filename, 1, 10},
   265  		{"//line foo::123abc\n", "invalid line number: 123abc", filename, 1, 13},
   266  		{"//line foo::0\n", "invalid line number: 0", filename, 1, 13},
   267  		{"//line foo:0:1\n", "invalid line number: 0", filename, 1, 12},
   268  
   269  		{"//line :123:0\n", "invalid column number: 0", filename, 1, 13},
   270  		{"//line foo:123:0\n", "invalid column number: 0", filename, 1, 16},
   271  		{fmt.Sprintf("//line foo:10:%d\n", tooLarge), fmt.Sprintf("invalid column number: %d", tooLarge), filename, 1, 15},
   272  
   273  		// effect of valid //line directives on lines
   274  		{"//line foo:123\n   foo", valid, "foo", 123, 0},
   275  		{"//line  foo:123\n   foo", valid, " foo", 123, 0},
   276  		{"//line foo:123\n//line bar:345\nfoo", valid, "bar", 345, 0},
   277  		{"//line C:foo:123\n", valid, "C:foo", 123, 0},
   278  		{"//line /src/a/a.go:123\n   foo", valid, "/src/a/a.go", 123, 0},
   279  		{"//line :x:1\n", valid, ":x", 1, 0},
   280  		{"//line foo ::1\n", valid, "foo :", 1, 0},
   281  		{"//line foo:123abc:1\n", valid, "foo:123abc", 1, 0},
   282  		{"//line foo :123:1\n", valid, "foo ", 123, 1},
   283  		{"//line ::123\n", valid, ":", 123, 0},
   284  
   285  		// effect of valid //line directives on columns
   286  		{"//line :x:1:10\n", valid, ":x", 1, 10},
   287  		{"//line foo ::1:2\n", valid, "foo :", 1, 2},
   288  		{"//line foo:123abc:1:1000\n", valid, "foo:123abc", 1, 1000},
   289  		{"//line foo :123:1000\n\n", valid, "foo ", 124, 1},
   290  		{"//line ::123:1234\n", valid, ":", 123, 1234},
   291  
   292  		// //line directives with omitted filenames lead to empty filenames
   293  		{"//line :10\n", valid, "", 10, 0},
   294  		{"//line :10:20\n", valid, filename, 10, 20},
   295  		{"//line bar:1\n//line :10\n", valid, "", 10, 0},
   296  		{"//line bar:1\n//line :10:20\n", valid, "bar", 10, 20},
   297  
   298  		// ignored /*line directives
   299  		{"/**/", valid, filename, 1, 5},             // no directive
   300  		{"/*line*/", valid, filename, 1, 9},         // missing colon
   301  		{"/*line foo*/", valid, filename, 1, 13},    // missing colon
   302  		{"  //line foo:*/", valid, filename, 1, 16}, // not a line start
   303  		{"/*  line foo:*/", valid, filename, 1, 16}, // space between // and line
   304  
   305  		// invalid /*line directives with one colon
   306  		{"/*line :*/", "invalid line number: ", filename, 1, 9},
   307  		{"/*line :x*/", "invalid line number: x", filename, 1, 9},
   308  		{"/*line foo :*/", "invalid line number: ", filename, 1, 13},
   309  		{"/*line foo:x*/", "invalid line number: x", filename, 1, 12},
   310  		{"/*line foo:0*/", "invalid line number: 0", filename, 1, 12},
   311  		{"/*line foo:1 */", "invalid line number: 1 ", filename, 1, 12},
   312  		{"/*line C:foo:0*/", "invalid line number: 0", filename, 1, 14},
   313  		{fmt.Sprintf("/*line foo:%d*/", tooLarge), fmt.Sprintf("invalid line number: %d", tooLarge), filename, 1, 12},
   314  
   315  		// invalid /*line directives with two colons
   316  		{"/*line ::*/", "invalid line number: ", filename, 1, 10},
   317  		{"/*line ::x*/", "invalid line number: x", filename, 1, 10},
   318  		{"/*line foo::123abc*/", "invalid line number: 123abc", filename, 1, 13},
   319  		{"/*line foo::0*/", "invalid line number: 0", filename, 1, 13},
   320  		{"/*line foo:0:1*/", "invalid line number: 0", filename, 1, 12},
   321  
   322  		{"/*line :123:0*/", "invalid column number: 0", filename, 1, 13},
   323  		{"/*line foo:123:0*/", "invalid column number: 0", filename, 1, 16},
   324  		{fmt.Sprintf("/*line foo:10:%d*/", tooLarge), fmt.Sprintf("invalid column number: %d", tooLarge), filename, 1, 15},
   325  
   326  		// effect of valid /*line directives on lines
   327  		{"/*line foo:123*/   foo", valid, "foo", 123, 0},
   328  		{"/*line foo:123*/\n//line bar:345\nfoo", valid, "bar", 345, 0},
   329  		{"/*line C:foo:123*/", valid, "C:foo", 123, 0},
   330  		{"/*line /src/a/a.go:123*/   foo", valid, "/src/a/a.go", 123, 0},
   331  		{"/*line :x:1*/", valid, ":x", 1, 0},
   332  		{"/*line foo ::1*/", valid, "foo :", 1, 0},
   333  		{"/*line foo:123abc:1*/", valid, "foo:123abc", 1, 0},
   334  		{"/*line foo :123:10*/", valid, "foo ", 123, 10},
   335  		{"/*line ::123*/", valid, ":", 123, 0},
   336  
   337  		// effect of valid /*line directives on columns
   338  		{"/*line :x:1:10*/", valid, ":x", 1, 10},
   339  		{"/*line foo ::1:2*/", valid, "foo :", 1, 2},
   340  		{"/*line foo:123abc:1:1000*/", valid, "foo:123abc", 1, 1000},
   341  		{"/*line foo :123:1000*/\n", valid, "foo ", 124, 1},
   342  		{"/*line ::123:1234*/", valid, ":", 123, 1234},
   343  
   344  		// /*line directives with omitted filenames lead to the previously used filenames
   345  		{"/*line :10*/", valid, "", 10, 0},
   346  		{"/*line :10:20*/", valid, filename, 10, 20},
   347  		{"//line bar:1\n/*line :10*/", valid, "", 10, 0},
   348  		{"//line bar:1\n/*line :10:20*/", valid, "bar", 10, 20},
   349  	} {
   350  		base := NewFileBase(filename)
   351  		_, err := Parse(base, strings.NewReader(test.src), nil, nil, 0)
   352  		if err == nil {
   353  			t.Errorf("%s: no error reported", test.src)
   354  			continue
   355  		}
   356  		perr, ok := err.(Error)
   357  		if !ok {
   358  			t.Errorf("%s: got %v; want parser error", test.src, err)
   359  			continue
   360  		}
   361  		if msg := perr.Msg; msg != test.msg {
   362  			t.Errorf("%s: got msg = %q; want %q", test.src, msg, test.msg)
   363  		}
   364  
   365  		pos := perr.Pos
   366  		if filename := pos.RelFilename(); filename != test.filename {
   367  			t.Errorf("%s: got filename = %q; want %q", test.src, filename, test.filename)
   368  		}
   369  		if line := pos.RelLine(); line != test.line {
   370  			t.Errorf("%s: got line = %d; want %d", test.src, line, test.line)
   371  		}
   372  		if col := pos.RelCol(); col != test.col {
   373  			t.Errorf("%s: got col = %d; want %d", test.src, col, test.col)
   374  		}
   375  	}
   376  }
   377  
   378  // Test that typical uses of UnpackListExpr don't allocate.
   379  func TestUnpackListExprAllocs(t *testing.T) {
   380  	var x Expr = NewName(Pos{}, "x")
   381  	allocs := testing.AllocsPerRun(1000, func() {
   382  		list := UnpackListExpr(x)
   383  		if len(list) != 1 || list[0] != x {
   384  			t.Fatalf("unexpected result")
   385  		}
   386  	})
   387  
   388  	if allocs > 0 {
   389  		errorf := t.Errorf
   390  		if testenv.OptimizationOff() {
   391  			errorf = t.Logf // noopt builder disables inlining
   392  		}
   393  		errorf("UnpackListExpr allocated %v times", allocs)
   394  	}
   395  }
   396
View as plain text