Source file src/net/textproto/reader_test.go

     1  // Copyright 2010 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package textproto
     6  
     7  import (
     8  	"bufio"
     9  	"bytes"
    10  	"io"
    11  	"net"
    12  	"reflect"
    13  	"runtime"
    14  	"strings"
    15  	"sync"
    16  	"testing"
    17  )
    18  
    19  func reader(s string) *Reader {
    20  	return NewReader(bufio.NewReader(strings.NewReader(s)))
    21  }
    22  
    23  func TestReadLine(t *testing.T) {
    24  	r := reader("line1\nline2\n")
    25  	s, err := r.ReadLine()
    26  	if s != "line1" || err != nil {
    27  		t.Fatalf("Line 1: %s, %v", s, err)
    28  	}
    29  	s, err = r.ReadLine()
    30  	if s != "line2" || err != nil {
    31  		t.Fatalf("Line 2: %s, %v", s, err)
    32  	}
    33  	s, err = r.ReadLine()
    34  	if s != "" || err != io.EOF {
    35  		t.Fatalf("EOF: %s, %v", s, err)
    36  	}
    37  }
    38  
    39  func TestReadLineLongLine(t *testing.T) {
    40  	line := strings.Repeat("12345", 10000)
    41  	r := reader(line + "\r\n")
    42  	s, err := r.ReadLine()
    43  	if err != nil {
    44  		t.Fatalf("Line 1: %v", err)
    45  	}
    46  	if s != line {
    47  		t.Fatalf("%v-byte line does not match expected %v-byte line", len(s), len(line))
    48  	}
    49  }
    50  
    51  func TestReadContinuedLine(t *testing.T) {
    52  	r := reader("line1\nline\n 2\nline3\n")
    53  	s, err := r.ReadContinuedLine()
    54  	if s != "line1" || err != nil {
    55  		t.Fatalf("Line 1: %s, %v", s, err)
    56  	}
    57  	s, err = r.ReadContinuedLine()
    58  	if s != "line 2" || err != nil {
    59  		t.Fatalf("Line 2: %s, %v", s, err)
    60  	}
    61  	s, err = r.ReadContinuedLine()
    62  	if s != "line3" || err != nil {
    63  		t.Fatalf("Line 3: %s, %v", s, err)
    64  	}
    65  	s, err = r.ReadContinuedLine()
    66  	if s != "" || err != io.EOF {
    67  		t.Fatalf("EOF: %s, %v", s, err)
    68  	}
    69  }
    70  
    71  func TestReadCodeLine(t *testing.T) {
    72  	r := reader("123 hi\n234 bye\n345 no way\n")
    73  	code, msg, err := r.ReadCodeLine(0)
    74  	if code != 123 || msg != "hi" || err != nil {
    75  		t.Fatalf("Line 1: %d, %s, %v", code, msg, err)
    76  	}
    77  	code, msg, err = r.ReadCodeLine(23)
    78  	if code != 234 || msg != "bye" || err != nil {
    79  		t.Fatalf("Line 2: %d, %s, %v", code, msg, err)
    80  	}
    81  	code, msg, err = r.ReadCodeLine(346)
    82  	if code != 345 || msg != "no way" || err == nil {
    83  		t.Fatalf("Line 3: %d, %s, %v", code, msg, err)
    84  	}
    85  	if e, ok := err.(*Error); !ok || e.Code != code || e.Msg != msg {
    86  		t.Fatalf("Line 3: wrong error %v\n", err)
    87  	}
    88  	code, msg, err = r.ReadCodeLine(1)
    89  	if code != 0 || msg != "" || err != io.EOF {
    90  		t.Fatalf("EOF: %d, %s, %v", code, msg, err)
    91  	}
    92  }
    93  
    94  func TestReadDotLines(t *testing.T) {
    95  	r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanother\n")
    96  	s, err := r.ReadDotLines()
    97  	want := []string{"dotlines", "foo", ".bar", "..baz", "quux", ""}
    98  	if !reflect.DeepEqual(s, want) || err != nil {
    99  		t.Fatalf("ReadDotLines: %v, %v", s, err)
   100  	}
   101  
   102  	s, err = r.ReadDotLines()
   103  	want = []string{"another"}
   104  	if !reflect.DeepEqual(s, want) || err != io.ErrUnexpectedEOF {
   105  		t.Fatalf("ReadDotLines2: %v, %v", s, err)
   106  	}
   107  }
   108  
   109  func TestReadDotBytes(t *testing.T) {
   110  	r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanot.her\r\n")
   111  	b, err := r.ReadDotBytes()
   112  	want := []byte("dotlines\nfoo\n.bar\n..baz\nquux\n\n")
   113  	if !reflect.DeepEqual(b, want) || err != nil {
   114  		t.Fatalf("ReadDotBytes: %q, %v", b, err)
   115  	}
   116  
   117  	b, err = r.ReadDotBytes()
   118  	want = []byte("anot.her\n")
   119  	if !reflect.DeepEqual(b, want) || err != io.ErrUnexpectedEOF {
   120  		t.Fatalf("ReadDotBytes2: %q, %v", b, err)
   121  	}
   122  }
   123  
   124  func TestReadMIMEHeader(t *testing.T) {
   125  	r := reader("my-key: Value 1  \r\nLong-key: Even \n Longer Value\r\nmy-Key: Value 2\r\n\n")
   126  	m, err := r.ReadMIMEHeader()
   127  	want := MIMEHeader{
   128  		"My-Key":   {"Value 1", "Value 2"},
   129  		"Long-Key": {"Even Longer Value"},
   130  	}
   131  	if !reflect.DeepEqual(m, want) || err != nil {
   132  		t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want)
   133  	}
   134  }
   135  
   136  func TestReadMIMEHeaderSingle(t *testing.T) {
   137  	r := reader("Foo: bar\n\n")
   138  	m, err := r.ReadMIMEHeader()
   139  	want := MIMEHeader{"Foo": {"bar"}}
   140  	if !reflect.DeepEqual(m, want) || err != nil {
   141  		t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want)
   142  	}
   143  }
   144  
   145  // TestReaderUpcomingHeaderKeys is testing an internal function, but it's very
   146  // difficult to test well via the external API.
   147  func TestReaderUpcomingHeaderKeys(t *testing.T) {
   148  	for _, test := range []struct {
   149  		input string
   150  		want  int
   151  	}{{
   152  		input: "",
   153  		want:  0,
   154  	}, {
   155  		input: "A: v",
   156  		want:  1,
   157  	}, {
   158  		input: "A: v\r\nB: v\r\n",
   159  		want:  2,
   160  	}, {
   161  		input: "A: v\nB: v\n",
   162  		want:  2,
   163  	}, {
   164  		input: "A: v\r\n  continued\r\n  still continued\r\nB: v\r\n\r\n",
   165  		want:  2,
   166  	}, {
   167  		input: "A: v\r\n\r\nB: v\r\nC: v\r\n",
   168  		want:  1,
   169  	}, {
   170  		input: "A: v" + strings.Repeat("\n", 1000),
   171  		want:  1,
   172  	}} {
   173  		r := reader(test.input)
   174  		got := r.upcomingHeaderKeys()
   175  		if test.want != got {
   176  			t.Fatalf("upcomingHeaderKeys(%q): %v; want %v", test.input, got, test.want)
   177  		}
   178  	}
   179  }
   180  
   181  func TestReadMIMEHeaderNoKey(t *testing.T) {
   182  	r := reader(": bar\ntest-1: 1\n\n")
   183  	m, err := r.ReadMIMEHeader()
   184  	want := MIMEHeader{"Test-1": {"1"}}
   185  	if !reflect.DeepEqual(m, want) || err != nil {
   186  		t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want)
   187  	}
   188  }
   189  
   190  func TestLargeReadMIMEHeader(t *testing.T) {
   191  	data := make([]byte, 16*1024)
   192  	for i := 0; i < len(data); i++ {
   193  		data[i] = 'x'
   194  	}
   195  	sdata := string(data)
   196  	r := reader("Cookie: " + sdata + "\r\n\n")
   197  	m, err := r.ReadMIMEHeader()
   198  	if err != nil {
   199  		t.Fatalf("ReadMIMEHeader: %v", err)
   200  	}
   201  	cookie := m.Get("Cookie")
   202  	if cookie != sdata {
   203  		t.Fatalf("ReadMIMEHeader: %v bytes, want %v bytes", len(cookie), len(sdata))
   204  	}
   205  }
   206  
   207  // TestReadMIMEHeaderNonCompliant checks that we don't normalize headers
   208  // with spaces before colons, and accept spaces in keys.
   209  func TestReadMIMEHeaderNonCompliant(t *testing.T) {
   210  	// These invalid headers will be rejected by net/http according to RFC 7230.
   211  	r := reader("Foo: bar\r\n" +
   212  		"Content-Language: en\r\n" +
   213  		"SID : 0\r\n" +
   214  		"Audio Mode : None\r\n" +
   215  		"Privilege : 127\r\n\r\n")
   216  	m, err := r.ReadMIMEHeader()
   217  	want := MIMEHeader{
   218  		"Foo":              {"bar"},
   219  		"Content-Language": {"en"},
   220  		"SID ":             {"0"},
   221  		"Audio Mode ":      {"None"},
   222  		"Privilege ":       {"127"},
   223  	}
   224  	if !reflect.DeepEqual(m, want) || err != nil {
   225  		t.Fatalf("ReadMIMEHeader =\n%v, %v; want:\n%v", m, err, want)
   226  	}
   227  }
   228  
   229  func TestReadMIMEHeaderMalformed(t *testing.T) {
   230  	inputs := []string{
   231  		"No colon first line\r\nFoo: foo\r\n\r\n",
   232  		" No colon first line with leading space\r\nFoo: foo\r\n\r\n",
   233  		"\tNo colon first line with leading tab\r\nFoo: foo\r\n\r\n",
   234  		" First: line with leading space\r\nFoo: foo\r\n\r\n",
   235  		"\tFirst: line with leading tab\r\nFoo: foo\r\n\r\n",
   236  		"Foo: foo\r\nNo colon second line\r\n\r\n",
   237  		"Foo-\n\tBar: foo\r\n\r\n",
   238  		"Foo-\r\n\tBar: foo\r\n\r\n",
   239  		"Foo\r\n\t: foo\r\n\r\n",
   240  		"Foo-\n\tBar",
   241  		"Foo \tBar: foo\r\n\r\n",
   242  	}
   243  	for _, input := range inputs {
   244  		r := reader(input)
   245  		if m, err := r.ReadMIMEHeader(); err == nil || err == io.EOF {
   246  			t.Errorf("ReadMIMEHeader(%q) = %v, %v; want nil, err", input, m, err)
   247  		}
   248  	}
   249  }
   250  
   251  func TestReadMIMEHeaderBytes(t *testing.T) {
   252  	for i := 0; i <= 0xff; i++ {
   253  		s := "Foo" + string(rune(i)) + "Bar: foo\r\n\r\n"
   254  		r := reader(s)
   255  		wantErr := true
   256  		switch {
   257  		case i >= '0' && i <= '9':
   258  			wantErr = false
   259  		case i >= 'a' && i <= 'z':
   260  			wantErr = false
   261  		case i >= 'A' && i <= 'Z':
   262  			wantErr = false
   263  		case i == '!' || i == '#' || i == '$' || i == '%' || i == '&' || i == '\'' || i == '*' || i == '+' || i == '-' || i == '.' || i == '^' || i == '_' || i == '`' || i == '|' || i == '~':
   264  			wantErr = false
   265  		case i == ':':
   266  			// Special case: "Foo:Bar: foo" is the header "Foo".
   267  			wantErr = false
   268  		case i == ' ':
   269  			wantErr = false
   270  		}
   271  		m, err := r.ReadMIMEHeader()
   272  		if err != nil != wantErr {
   273  			t.Errorf("ReadMIMEHeader(%q) = %v, %v; want error=%v", s, m, err, wantErr)
   274  		}
   275  	}
   276  	for i := 0; i <= 0xff; i++ {
   277  		s := "Foo: foo" + string(rune(i)) + "bar\r\n\r\n"
   278  		r := reader(s)
   279  		wantErr := true
   280  		switch {
   281  		case i >= 0x21 && i <= 0x7e:
   282  			wantErr = false
   283  		case i == ' ':
   284  			wantErr = false
   285  		case i == '\t':
   286  			wantErr = false
   287  		case i >= 0x80 && i <= 0xff:
   288  			wantErr = false
   289  		}
   290  		m, err := r.ReadMIMEHeader()
   291  		if (err != nil) != wantErr {
   292  			t.Errorf("ReadMIMEHeader(%q) = %v, %v; want error=%v", s, m, err, wantErr)
   293  		}
   294  	}
   295  }
   296  
   297  // Test that continued lines are properly trimmed. Issue 11204.
   298  func TestReadMIMEHeaderTrimContinued(t *testing.T) {
   299  	// In this header, \n and \r\n terminated lines are mixed on purpose.
   300  	// We expect each line to be trimmed (prefix and suffix) before being concatenated.
   301  	// Keep the spaces as they are.
   302  	r := reader("" + // for code formatting purpose.
   303  		"a:\n" +
   304  		" 0 \r\n" +
   305  		"b:1 \t\r\n" +
   306  		"c: 2\r\n" +
   307  		" 3\t\n" +
   308  		"  \t 4  \r\n\n")
   309  	m, err := r.ReadMIMEHeader()
   310  	if err != nil {
   311  		t.Fatal(err)
   312  	}
   313  	want := MIMEHeader{
   314  		"A": {"0"},
   315  		"B": {"1"},
   316  		"C": {"2 3 4"},
   317  	}
   318  	if !reflect.DeepEqual(m, want) {
   319  		t.Fatalf("ReadMIMEHeader mismatch.\n got: %q\nwant: %q", m, want)
   320  	}
   321  }
   322  
   323  // Test that reading a header doesn't overallocate. Issue 58975.
   324  func TestReadMIMEHeaderAllocations(t *testing.T) {
   325  	var totalAlloc uint64
   326  	const count = 200
   327  	for i := 0; i < count; i++ {
   328  		r := reader("A: b\r\n\r\n" + strings.Repeat("\n", 4096))
   329  		var m1, m2 runtime.MemStats
   330  		runtime.ReadMemStats(&m1)
   331  		_, err := r.ReadMIMEHeader()
   332  		if err != nil {
   333  			t.Fatalf("ReadMIMEHeader: %v", err)
   334  		}
   335  		runtime.ReadMemStats(&m2)
   336  		totalAlloc += m2.TotalAlloc - m1.TotalAlloc
   337  	}
   338  	// 32k is large and we actually allocate substantially less,
   339  	// but prior to the fix for #58975 we allocated ~400k in this case.
   340  	if got, want := totalAlloc/count, uint64(32768); got > want {
   341  		t.Fatalf("ReadMIMEHeader allocated %v bytes, want < %v", got, want)
   342  	}
   343  }
   344  
   345  type readResponseTest struct {
   346  	in       string
   347  	inCode   int
   348  	wantCode int
   349  	wantMsg  string
   350  }
   351  
   352  var readResponseTests = []readResponseTest{
   353  	{"230-Anonymous access granted, restrictions apply\n" +
   354  		"Read the file README.txt,\n" +
   355  		"230  please",
   356  		23,
   357  		230,
   358  		"Anonymous access granted, restrictions apply\nRead the file README.txt,\n please",
   359  	},
   360  
   361  	{"230 Anonymous access granted, restrictions apply\n",
   362  		23,
   363  		230,
   364  		"Anonymous access granted, restrictions apply",
   365  	},
   366  
   367  	{"400-A\n400-B\n400 C",
   368  		4,
   369  		400,
   370  		"A\nB\nC",
   371  	},
   372  
   373  	{"400-A\r\n400-B\r\n400 C\r\n",
   374  		4,
   375  		400,
   376  		"A\nB\nC",
   377  	},
   378  }
   379  
   380  // See https://www.ietf.org/rfc/rfc959.txt page 36.
   381  func TestRFC959Lines(t *testing.T) {
   382  	for i, tt := range readResponseTests {
   383  		r := reader(tt.in + "\nFOLLOWING DATA")
   384  		code, msg, err := r.ReadResponse(tt.inCode)
   385  		if err != nil {
   386  			t.Errorf("#%d: ReadResponse: %v", i, err)
   387  			continue
   388  		}
   389  		if code != tt.wantCode {
   390  			t.Errorf("#%d: code=%d, want %d", i, code, tt.wantCode)
   391  		}
   392  		if msg != tt.wantMsg {
   393  			t.Errorf("#%d: msg=%q, want %q", i, msg, tt.wantMsg)
   394  		}
   395  	}
   396  }
   397  
   398  // Test that multi-line errors are appropriately and fully read. Issue 10230.
   399  func TestReadMultiLineError(t *testing.T) {
   400  	r := reader("550-5.1.1 The email account that you tried to reach does not exist. Please try\n" +
   401  		"550-5.1.1 double-checking the recipient's email address for typos or\n" +
   402  		"550-5.1.1 unnecessary spaces. Learn more at\n" +
   403  		"Unexpected but legal text!\n" +
   404  		"550 5.1.1 https://support.google.com/mail/answer/6596 h20si25154304pfd.166 - gsmtp\n")
   405  
   406  	wantMsg := "5.1.1 The email account that you tried to reach does not exist. Please try\n" +
   407  		"5.1.1 double-checking the recipient's email address for typos or\n" +
   408  		"5.1.1 unnecessary spaces. Learn more at\n" +
   409  		"Unexpected but legal text!\n" +
   410  		"5.1.1 https://support.google.com/mail/answer/6596 h20si25154304pfd.166 - gsmtp"
   411  
   412  	code, msg, err := r.ReadResponse(250)
   413  	if err == nil {
   414  		t.Errorf("ReadResponse: no error, want error")
   415  	}
   416  	if code != 550 {
   417  		t.Errorf("ReadResponse: code=%d, want %d", code, 550)
   418  	}
   419  	if msg != wantMsg {
   420  		t.Errorf("ReadResponse: msg=%q, want %q", msg, wantMsg)
   421  	}
   422  	if err != nil && err.Error() != "550 "+wantMsg {
   423  		t.Errorf("ReadResponse: error=%q, want %q", err.Error(), "550 "+wantMsg)
   424  	}
   425  }
   426  
   427  func TestCommonHeaders(t *testing.T) {
   428  	commonHeaderOnce.Do(initCommonHeader)
   429  	for h := range commonHeader {
   430  		if h != CanonicalMIMEHeaderKey(h) {
   431  			t.Errorf("Non-canonical header %q in commonHeader", h)
   432  		}
   433  	}
   434  	b := []byte("content-Length")
   435  	want := "Content-Length"
   436  	n := testing.AllocsPerRun(200, func() {
   437  		if x, _ := canonicalMIMEHeaderKey(b); x != want {
   438  			t.Fatalf("canonicalMIMEHeaderKey(%q) = %q; want %q", b, x, want)
   439  		}
   440  	})
   441  	if n > 0 {
   442  		t.Errorf("canonicalMIMEHeaderKey allocs = %v; want 0", n)
   443  	}
   444  }
   445  
   446  func TestIssue46363(t *testing.T) {
   447  	// Regression test for data race reported in issue 46363:
   448  	// ReadMIMEHeader reads commonHeader before commonHeader has been initialized.
   449  	// Run this test with the race detector enabled to catch the reported data race.
   450  
   451  	// Reset commonHeaderOnce, so that commonHeader will have to be initialized
   452  	commonHeaderOnce = sync.Once{}
   453  	commonHeader = nil
   454  
   455  	// Test for data race by calling ReadMIMEHeader and CanonicalMIMEHeaderKey concurrently
   456  
   457  	// Send MIME header over net.Conn
   458  	r, w := net.Pipe()
   459  	go func() {
   460  		// ReadMIMEHeader calls canonicalMIMEHeaderKey, which reads from commonHeader
   461  		NewConn(r).ReadMIMEHeader()
   462  	}()
   463  	w.Write([]byte("A: 1\r\nB: 2\r\nC: 3\r\n\r\n"))
   464  
   465  	// CanonicalMIMEHeaderKey calls commonHeaderOnce.Do(initCommonHeader) which initializes commonHeader
   466  	CanonicalMIMEHeaderKey("a")
   467  
   468  	if commonHeader == nil {
   469  		t.Fatal("CanonicalMIMEHeaderKey should initialize commonHeader")
   470  	}
   471  }
   472  
   473  var clientHeaders = strings.Replace(`Host: golang.org
   474  Connection: keep-alive
   475  Cache-Control: max-age=0
   476  Accept: application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
   477  User-Agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3
   478  Accept-Encoding: gzip,deflate,sdch
   479  Accept-Language: en-US,en;q=0.8,fr-CH;q=0.6
   480  Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3
   481  COOKIE: __utma=000000000.0000000000.0000000000.0000000000.0000000000.00; __utmb=000000000.0.00.0000000000; __utmc=000000000; __utmz=000000000.0000000000.00.0.utmcsr=code.google.com|utmccn=(referral)|utmcmd=referral|utmcct=/p/go/issues/detail
   482  Non-Interned: test
   483  
   484  `, "\n", "\r\n", -1)
   485  
   486  var serverHeaders = strings.Replace(`Content-Type: text/html; charset=utf-8
   487  Content-Encoding: gzip
   488  Date: Thu, 27 Sep 2012 09:03:33 GMT
   489  Server: Google Frontend
   490  Cache-Control: private
   491  Content-Length: 2298
   492  VIA: 1.1 proxy.example.com:80 (XXX/n.n.n-nnn)
   493  Connection: Close
   494  Non-Interned: test
   495  
   496  `, "\n", "\r\n", -1)
   497  
   498  func BenchmarkReadMIMEHeader(b *testing.B) {
   499  	b.ReportAllocs()
   500  	for _, set := range []struct {
   501  		name    string
   502  		headers string
   503  	}{
   504  		{"client_headers", clientHeaders},
   505  		{"server_headers", serverHeaders},
   506  	} {
   507  		b.Run(set.name, func(b *testing.B) {
   508  			var buf bytes.Buffer
   509  			br := bufio.NewReader(&buf)
   510  			r := NewReader(br)
   511  
   512  			for i := 0; i < b.N; i++ {
   513  				buf.WriteString(set.headers)
   514  				if _, err := r.ReadMIMEHeader(); err != nil {
   515  					b.Fatal(err)
   516  				}
   517  			}
   518  		})
   519  	}
   520  }
   521  
   522  func BenchmarkUncommon(b *testing.B) {
   523  	b.ReportAllocs()
   524  	var buf bytes.Buffer
   525  	br := bufio.NewReader(&buf)
   526  	r := NewReader(br)
   527  	for i := 0; i < b.N; i++ {
   528  		buf.WriteString("uncommon-header-for-benchmark: foo\r\n\r\n")
   529  		h, err := r.ReadMIMEHeader()
   530  		if err != nil {
   531  			b.Fatal(err)
   532  		}
   533  		if _, ok := h["Uncommon-Header-For-Benchmark"]; !ok {
   534  			b.Fatal("Missing result header.")
   535  		}
   536  	}
   537  }
   538  

View as plain text