Source file src/net/http/client.go
1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // HTTP client. See RFC 7230 through 7235. 6 // 7 // This is the high-level Client interface. 8 // The low-level implementation is in transport.go. 9 10 package http 11 12 import ( 13 "context" 14 "crypto/tls" 15 "encoding/base64" 16 "errors" 17 "fmt" 18 "io" 19 "log" 20 "net/http/internal/ascii" 21 "net/url" 22 "reflect" 23 "sort" 24 "strings" 25 "sync" 26 "sync/atomic" 27 "time" 28 ) 29 30 // A Client is an HTTP client. Its zero value (DefaultClient) is a 31 // usable client that uses DefaultTransport. 32 // 33 // The Client's Transport typically has internal state (cached TCP 34 // connections), so Clients should be reused instead of created as 35 // needed. Clients are safe for concurrent use by multiple goroutines. 36 // 37 // A Client is higher-level than a RoundTripper (such as Transport) 38 // and additionally handles HTTP details such as cookies and 39 // redirects. 40 // 41 // When following redirects, the Client will forward all headers set on the 42 // initial Request except: 43 // 44 // • when forwarding sensitive headers like "Authorization", 45 // "WWW-Authenticate", and "Cookie" to untrusted targets. 46 // These headers will be ignored when following a redirect to a domain 47 // that is not a subdomain match or exact match of the initial domain. 48 // For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" 49 // will forward the sensitive headers, but a redirect to "bar.com" will not. 50 // 51 // • when forwarding the "Cookie" header with a non-nil cookie Jar. 52 // Since each redirect may mutate the state of the cookie jar, 53 // a redirect may possibly alter a cookie set in the initial request. 54 // When forwarding the "Cookie" header, any mutated cookies will be omitted, 55 // with the expectation that the Jar will insert those mutated cookies 56 // with the updated values (assuming the origin matches). 57 // If Jar is nil, the initial cookies are forwarded without change. 58 type Client struct { 59 // Transport specifies the mechanism by which individual 60 // HTTP requests are made. 61 // If nil, DefaultTransport is used. 62 Transport RoundTripper 63 64 // CheckRedirect specifies the policy for handling redirects. 65 // If CheckRedirect is not nil, the client calls it before 66 // following an HTTP redirect. The arguments req and via are 67 // the upcoming request and the requests made already, oldest 68 // first. If CheckRedirect returns an error, the Client's Get 69 // method returns both the previous Response (with its Body 70 // closed) and CheckRedirect's error (wrapped in a url.Error) 71 // instead of issuing the Request req. 72 // As a special case, if CheckRedirect returns ErrUseLastResponse, 73 // then the most recent response is returned with its body 74 // unclosed, along with a nil error. 75 // 76 // If CheckRedirect is nil, the Client uses its default policy, 77 // which is to stop after 10 consecutive requests. 78 CheckRedirect func(req *Request, via []*Request) error 79 80 // Jar specifies the cookie jar. 81 // 82 // The Jar is used to insert relevant cookies into every 83 // outbound Request and is updated with the cookie values 84 // of every inbound Response. The Jar is consulted for every 85 // redirect that the Client follows. 86 // 87 // If Jar is nil, cookies are only sent if they are explicitly 88 // set on the Request. 89 Jar CookieJar 90 91 // Timeout specifies a time limit for requests made by this 92 // Client. The timeout includes connection time, any 93 // redirects, and reading the response body. The timer remains 94 // running after Get, Head, Post, or Do return and will 95 // interrupt reading of the Response.Body. 96 // 97 // A Timeout of zero means no timeout. 98 // 99 // The Client cancels requests to the underlying Transport 100 // as if the Request's Context ended. 101 // 102 // For compatibility, the Client will also use the deprecated 103 // CancelRequest method on Transport if found. New 104 // RoundTripper implementations should use the Request's Context 105 // for cancellation instead of implementing CancelRequest. 106 Timeout time.Duration 107 } 108 109 // DefaultClient is the default Client and is used by Get, Head, and Post. 110 var DefaultClient = &Client{} 111 112 // RoundTripper is an interface representing the ability to execute a 113 // single HTTP transaction, obtaining the Response for a given Request. 114 // 115 // A RoundTripper must be safe for concurrent use by multiple 116 // goroutines. 117 type RoundTripper interface { 118 // RoundTrip executes a single HTTP transaction, returning 119 // a Response for the provided Request. 120 // 121 // RoundTrip should not attempt to interpret the response. In 122 // particular, RoundTrip must return err == nil if it obtained 123 // a response, regardless of the response's HTTP status code. 124 // A non-nil err should be reserved for failure to obtain a 125 // response. Similarly, RoundTrip should not attempt to 126 // handle higher-level protocol details such as redirects, 127 // authentication, or cookies. 128 // 129 // RoundTrip should not modify the request, except for 130 // consuming and closing the Request's Body. RoundTrip may 131 // read fields of the request in a separate goroutine. Callers 132 // should not mutate or reuse the request until the Response's 133 // Body has been closed. 134 // 135 // RoundTrip must always close the body, including on errors, 136 // but depending on the implementation may do so in a separate 137 // goroutine even after RoundTrip returns. This means that 138 // callers wanting to reuse the body for subsequent requests 139 // must arrange to wait for the Close call before doing so. 140 // 141 // The Request's URL and Header fields must be initialized. 142 RoundTrip(*Request) (*Response, error) 143 } 144 145 // refererForURL returns a referer without any authentication info or 146 // an empty string if lastReq scheme is https and newReq scheme is http. 147 func refererForURL(lastReq, newReq *url.URL) string { 148 // https://tools.ietf.org/html/rfc7231#section-5.5.2 149 // "Clients SHOULD NOT include a Referer header field in a 150 // (non-secure) HTTP request if the referring page was 151 // transferred with a secure protocol." 152 if lastReq.Scheme == "https" && newReq.Scheme == "http" { 153 return "" 154 } 155 referer := lastReq.String() 156 if lastReq.User != nil { 157 // This is not very efficient, but is the best we can 158 // do without: 159 // - introducing a new method on URL 160 // - creating a race condition 161 // - copying the URL struct manually, which would cause 162 // maintenance problems down the line 163 auth := lastReq.User.String() + "@" 164 referer = strings.Replace(referer, auth, "", 1) 165 } 166 return referer 167 } 168 169 // didTimeout is non-nil only if err != nil. 170 func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { 171 if c.Jar != nil { 172 for _, cookie := range c.Jar.Cookies(req.URL) { 173 req.AddCookie(cookie) 174 } 175 } 176 resp, didTimeout, err = send(req, c.transport(), deadline) 177 if err != nil { 178 return nil, didTimeout, err 179 } 180 if c.Jar != nil { 181 if rc := resp.Cookies(); len(rc) > 0 { 182 c.Jar.SetCookies(req.URL, rc) 183 } 184 } 185 return resp, nil, nil 186 } 187 188 func (c *Client) deadline() time.Time { 189 if c.Timeout > 0 { 190 return time.Now().Add(c.Timeout) 191 } 192 return time.Time{} 193 } 194 195 func (c *Client) transport() RoundTripper { 196 if c.Transport != nil { 197 return c.Transport 198 } 199 return DefaultTransport 200 } 201 202 // send issues an HTTP request. 203 // Caller should close resp.Body when done reading from it. 204 func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { 205 req := ireq // req is either the original request, or a modified fork 206 207 if rt == nil { 208 req.closeBody() 209 return nil, alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport") 210 } 211 212 if req.URL == nil { 213 req.closeBody() 214 return nil, alwaysFalse, errors.New("http: nil Request.URL") 215 } 216 217 if req.RequestURI != "" { 218 req.closeBody() 219 return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests") 220 } 221 222 // forkReq forks req into a shallow clone of ireq the first 223 // time it's called. 224 forkReq := func() { 225 if ireq == req { 226 req = new(Request) 227 *req = *ireq // shallow clone 228 } 229 } 230 231 // Most the callers of send (Get, Post, et al) don't need 232 // Headers, leaving it uninitialized. We guarantee to the 233 // Transport that this has been initialized, though. 234 if req.Header == nil { 235 forkReq() 236 req.Header = make(Header) 237 } 238 239 if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" { 240 username := u.Username() 241 password, _ := u.Password() 242 forkReq() 243 req.Header = cloneOrMakeHeader(ireq.Header) 244 req.Header.Set("Authorization", "Basic "+basicAuth(username, password)) 245 } 246 247 if !deadline.IsZero() { 248 forkReq() 249 } 250 stopTimer, didTimeout := setRequestCancel(req, rt, deadline) 251 252 resp, err = rt.RoundTrip(req) 253 if err != nil { 254 stopTimer() 255 if resp != nil { 256 log.Printf("RoundTripper returned a response & error; ignoring response") 257 } 258 if tlsErr, ok := err.(tls.RecordHeaderError); ok { 259 // If we get a bad TLS record header, check to see if the 260 // response looks like HTTP and give a more helpful error. 261 // See golang.org/issue/11111. 262 if string(tlsErr.RecordHeader[:]) == "HTTP/" { 263 err = errors.New("http: server gave HTTP response to HTTPS client") 264 } 265 } 266 return nil, didTimeout, err 267 } 268 if resp == nil { 269 return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a nil *Response with a nil error", rt) 270 } 271 if resp.Body == nil { 272 // The documentation on the Body field says “The http Client and Transport 273 // guarantee that Body is always non-nil, even on responses without a body 274 // or responses with a zero-length body.” Unfortunately, we didn't document 275 // that same constraint for arbitrary RoundTripper implementations, and 276 // RoundTripper implementations in the wild (mostly in tests) assume that 277 // they can use a nil Body to mean an empty one (similar to Request.Body). 278 // (See https://golang.org/issue/38095.) 279 // 280 // If the ContentLength allows the Body to be empty, fill in an empty one 281 // here to ensure that it is non-nil. 282 if resp.ContentLength > 0 && req.Method != "HEAD" { 283 return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a *Response with content length %d but a nil Body", rt, resp.ContentLength) 284 } 285 resp.Body = io.NopCloser(strings.NewReader("")) 286 } 287 if !deadline.IsZero() { 288 resp.Body = &cancelTimerBody{ 289 stop: stopTimer, 290 rc: resp.Body, 291 reqDidTimeout: didTimeout, 292 } 293 } 294 return resp, nil, nil 295 } 296 297 // timeBeforeContextDeadline reports whether the non-zero Time t is 298 // before ctx's deadline, if any. If ctx does not have a deadline, it 299 // always reports true (the deadline is considered infinite). 300 func timeBeforeContextDeadline(t time.Time, ctx context.Context) bool { 301 d, ok := ctx.Deadline() 302 if !ok { 303 return true 304 } 305 return t.Before(d) 306 } 307 308 // knownRoundTripperImpl reports whether rt is a RoundTripper that's 309 // maintained by the Go team and known to implement the latest 310 // optional semantics (notably contexts). The Request is used 311 // to check whether this particular request is using an alternate protocol, 312 // in which case we need to check the RoundTripper for that protocol. 313 func knownRoundTripperImpl(rt RoundTripper, req *Request) bool { 314 switch t := rt.(type) { 315 case *Transport: 316 if altRT := t.alternateRoundTripper(req); altRT != nil { 317 return knownRoundTripperImpl(altRT, req) 318 } 319 return true 320 case *http2Transport, http2noDialH2RoundTripper: 321 return true 322 } 323 // There's a very minor chance of a false positive with this. 324 // Instead of detecting our golang.org/x/net/http2.Transport, 325 // it might detect a Transport type in a different http2 326 // package. But I know of none, and the only problem would be 327 // some temporarily leaked goroutines if the transport didn't 328 // support contexts. So this is a good enough heuristic: 329 if reflect.TypeOf(rt).String() == "*http2.Transport" { 330 return true 331 } 332 return false 333 } 334 335 // setRequestCancel sets req.Cancel and adds a deadline context to req 336 // if deadline is non-zero. The RoundTripper's type is used to 337 // determine whether the legacy CancelRequest behavior should be used. 338 // 339 // As background, there are three ways to cancel a request: 340 // First was Transport.CancelRequest. (deprecated) 341 // Second was Request.Cancel. 342 // Third was Request.Context. 343 // This function populates the second and third, and uses the first if it really needs to. 344 func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) { 345 if deadline.IsZero() { 346 return nop, alwaysFalse 347 } 348 knownTransport := knownRoundTripperImpl(rt, req) 349 oldCtx := req.Context() 350 351 if req.Cancel == nil && knownTransport { 352 // If they already had a Request.Context that's 353 // expiring sooner, do nothing: 354 if !timeBeforeContextDeadline(deadline, oldCtx) { 355 return nop, alwaysFalse 356 } 357 358 var cancelCtx func() 359 req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) 360 return cancelCtx, func() bool { return time.Now().After(deadline) } 361 } 362 initialReqCancel := req.Cancel // the user's original Request.Cancel, if any 363 364 var cancelCtx func() 365 if timeBeforeContextDeadline(deadline, oldCtx) { 366 req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) 367 } 368 369 cancel := make(chan struct{}) 370 req.Cancel = cancel 371 372 doCancel := func() { 373 // The second way in the func comment above: 374 close(cancel) 375 // The first way, used only for RoundTripper 376 // implementations written before Go 1.5 or Go 1.6. 377 type canceler interface{ CancelRequest(*Request) } 378 if v, ok := rt.(canceler); ok { 379 v.CancelRequest(req) 380 } 381 } 382 383 stopTimerCh := make(chan struct{}) 384 var once sync.Once 385 stopTimer = func() { 386 once.Do(func() { 387 close(stopTimerCh) 388 if cancelCtx != nil { 389 cancelCtx() 390 } 391 }) 392 } 393 394 timer := time.NewTimer(time.Until(deadline)) 395 var timedOut atomic.Bool 396 397 go func() { 398 select { 399 case <-initialReqCancel: 400 doCancel() 401 timer.Stop() 402 case <-timer.C: 403 timedOut.Store(true) 404 doCancel() 405 case <-stopTimerCh: 406 timer.Stop() 407 } 408 }() 409 410 return stopTimer, timedOut.Load 411 } 412 413 // See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt 414 // "To receive authorization, the client sends the userid and password, 415 // separated by a single colon (":") character, within a base64 416 // encoded string in the credentials." 417 // It is not meant to be urlencoded. 418 func basicAuth(username, password string) string { 419 auth := username + ":" + password 420 return base64.StdEncoding.EncodeToString([]byte(auth)) 421 } 422 423 // Get issues a GET to the specified URL. If the response is one of 424 // the following redirect codes, Get follows the redirect, up to a 425 // maximum of 10 redirects: 426 // 427 // 301 (Moved Permanently) 428 // 302 (Found) 429 // 303 (See Other) 430 // 307 (Temporary Redirect) 431 // 308 (Permanent Redirect) 432 // 433 // An error is returned if there were too many redirects or if there 434 // was an HTTP protocol error. A non-2xx response doesn't cause an 435 // error. Any returned error will be of type *url.Error. The url.Error 436 // value's Timeout method will report true if the request timed out. 437 // 438 // When err is nil, resp always contains a non-nil resp.Body. 439 // Caller should close resp.Body when done reading from it. 440 // 441 // Get is a wrapper around DefaultClient.Get. 442 // 443 // To make a request with custom headers, use NewRequest and 444 // DefaultClient.Do. 445 // 446 // To make a request with a specified context.Context, use NewRequestWithContext 447 // and DefaultClient.Do. 448 func Get(url string) (resp *Response, err error) { 449 return DefaultClient.Get(url) 450 } 451 452 // Get issues a GET to the specified URL. If the response is one of the 453 // following redirect codes, Get follows the redirect after calling the 454 // Client's CheckRedirect function: 455 // 456 // 301 (Moved Permanently) 457 // 302 (Found) 458 // 303 (See Other) 459 // 307 (Temporary Redirect) 460 // 308 (Permanent Redirect) 461 // 462 // An error is returned if the Client's CheckRedirect function fails 463 // or if there was an HTTP protocol error. A non-2xx response doesn't 464 // cause an error. Any returned error will be of type *url.Error. The 465 // url.Error value's Timeout method will report true if the request 466 // timed out. 467 // 468 // When err is nil, resp always contains a non-nil resp.Body. 469 // Caller should close resp.Body when done reading from it. 470 // 471 // To make a request with custom headers, use NewRequest and Client.Do. 472 // 473 // To make a request with a specified context.Context, use NewRequestWithContext 474 // and Client.Do. 475 func (c *Client) Get(url string) (resp *Response, err error) { 476 req, err := NewRequest("GET", url, nil) 477 if err != nil { 478 return nil, err 479 } 480 return c.Do(req) 481 } 482 483 func alwaysFalse() bool { return false } 484 485 // ErrUseLastResponse can be returned by Client.CheckRedirect hooks to 486 // control how redirects are processed. If returned, the next request 487 // is not sent and the most recent response is returned with its body 488 // unclosed. 489 var ErrUseLastResponse = errors.New("net/http: use last response") 490 491 // checkRedirect calls either the user's configured CheckRedirect 492 // function, or the default. 493 func (c *Client) checkRedirect(req *Request, via []*Request) error { 494 fn := c.CheckRedirect 495 if fn == nil { 496 fn = defaultCheckRedirect 497 } 498 return fn(req, via) 499 } 500 501 // redirectBehavior describes what should happen when the 502 // client encounters a 3xx status code from the server. 503 func redirectBehavior(reqMethod string, resp *Response, ireq *Request) (redirectMethod string, shouldRedirect, includeBody bool) { 504 switch resp.StatusCode { 505 case 301, 302, 303: 506 redirectMethod = reqMethod 507 shouldRedirect = true 508 includeBody = false 509 510 // RFC 2616 allowed automatic redirection only with GET and 511 // HEAD requests. RFC 7231 lifts this restriction, but we still 512 // restrict other methods to GET to maintain compatibility. 513 // See Issue 18570. 514 if reqMethod != "GET" && reqMethod != "HEAD" { 515 redirectMethod = "GET" 516 } 517 case 307, 308: 518 redirectMethod = reqMethod 519 shouldRedirect = true 520 includeBody = true 521 522 if ireq.GetBody == nil && ireq.outgoingLength() != 0 { 523 // We had a request body, and 307/308 require 524 // re-sending it, but GetBody is not defined. So just 525 // return this response to the user instead of an 526 // error, like we did in Go 1.7 and earlier. 527 shouldRedirect = false 528 } 529 } 530 return redirectMethod, shouldRedirect, includeBody 531 } 532 533 // urlErrorOp returns the (*url.Error).Op value to use for the 534 // provided (*Request).Method value. 535 func urlErrorOp(method string) string { 536 if method == "" { 537 return "Get" 538 } 539 if lowerMethod, ok := ascii.ToLower(method); ok { 540 return method[:1] + lowerMethod[1:] 541 } 542 return method 543 } 544 545 // Do sends an HTTP request and returns an HTTP response, following 546 // policy (such as redirects, cookies, auth) as configured on the 547 // client. 548 // 549 // An error is returned if caused by client policy (such as 550 // CheckRedirect), or failure to speak HTTP (such as a network 551 // connectivity problem). A non-2xx status code doesn't cause an 552 // error. 553 // 554 // If the returned error is nil, the Response will contain a non-nil 555 // Body which the user is expected to close. If the Body is not both 556 // read to EOF and closed, the Client's underlying RoundTripper 557 // (typically Transport) may not be able to re-use a persistent TCP 558 // connection to the server for a subsequent "keep-alive" request. 559 // 560 // The request Body, if non-nil, will be closed by the underlying 561 // Transport, even on errors. 562 // 563 // On error, any Response can be ignored. A non-nil Response with a 564 // non-nil error only occurs when CheckRedirect fails, and even then 565 // the returned Response.Body is already closed. 566 // 567 // Generally Get, Post, or PostForm will be used instead of Do. 568 // 569 // If the server replies with a redirect, the Client first uses the 570 // CheckRedirect function to determine whether the redirect should be 571 // followed. If permitted, a 301, 302, or 303 redirect causes 572 // subsequent requests to use HTTP method GET 573 // (or HEAD if the original request was HEAD), with no body. 574 // A 307 or 308 redirect preserves the original HTTP method and body, 575 // provided that the Request.GetBody function is defined. 576 // The NewRequest function automatically sets GetBody for common 577 // standard library body types. 578 // 579 // Any returned error will be of type *url.Error. The url.Error 580 // value's Timeout method will report true if the request timed out. 581 func (c *Client) Do(req *Request) (*Response, error) { 582 return c.do(req) 583 } 584 585 var testHookClientDoResult func(retres *Response, reterr error) 586 587 func (c *Client) do(req *Request) (retres *Response, reterr error) { 588 if testHookClientDoResult != nil { 589 defer func() { testHookClientDoResult(retres, reterr) }() 590 } 591 if req.URL == nil { 592 req.closeBody() 593 return nil, &url.Error{ 594 Op: urlErrorOp(req.Method), 595 Err: errors.New("http: nil Request.URL"), 596 } 597 } 598 599 var ( 600 deadline = c.deadline() 601 reqs []*Request 602 resp *Response 603 copyHeaders = c.makeHeadersCopier(req) 604 reqBodyClosed = false // have we closed the current req.Body? 605 606 // Redirect behavior: 607 redirectMethod string 608 includeBody bool 609 ) 610 uerr := func(err error) error { 611 // the body may have been closed already by c.send() 612 if !reqBodyClosed { 613 req.closeBody() 614 } 615 var urlStr string 616 if resp != nil && resp.Request != nil { 617 urlStr = stripPassword(resp.Request.URL) 618 } else { 619 urlStr = stripPassword(req.URL) 620 } 621 return &url.Error{ 622 Op: urlErrorOp(reqs[0].Method), 623 URL: urlStr, 624 Err: err, 625 } 626 } 627 for { 628 // For all but the first request, create the next 629 // request hop and replace req. 630 if len(reqs) > 0 { 631 loc := resp.Header.Get("Location") 632 if loc == "" { 633 // While most 3xx responses include a Location, it is not 634 // required and 3xx responses without a Location have been 635 // observed in the wild. See issues #17773 and #49281. 636 return resp, nil 637 } 638 u, err := req.URL.Parse(loc) 639 if err != nil { 640 resp.closeBody() 641 return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err)) 642 } 643 host := "" 644 if req.Host != "" && req.Host != req.URL.Host { 645 // If the caller specified a custom Host header and the 646 // redirect location is relative, preserve the Host header 647 // through the redirect. See issue #22233. 648 if u, _ := url.Parse(loc); u != nil && !u.IsAbs() { 649 host = req.Host 650 } 651 } 652 ireq := reqs[0] 653 req = &Request{ 654 Method: redirectMethod, 655 Response: resp, 656 URL: u, 657 Header: make(Header), 658 Host: host, 659 Cancel: ireq.Cancel, 660 ctx: ireq.ctx, 661 } 662 if includeBody && ireq.GetBody != nil { 663 req.Body, err = ireq.GetBody() 664 if err != nil { 665 resp.closeBody() 666 return nil, uerr(err) 667 } 668 req.ContentLength = ireq.ContentLength 669 } 670 671 // Copy original headers before setting the Referer, 672 // in case the user set Referer on their first request. 673 // If they really want to override, they can do it in 674 // their CheckRedirect func. 675 copyHeaders(req) 676 677 // Add the Referer header from the most recent 678 // request URL to the new one, if it's not https->http: 679 if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL); ref != "" { 680 req.Header.Set("Referer", ref) 681 } 682 err = c.checkRedirect(req, reqs) 683 684 // Sentinel error to let users select the 685 // previous response, without closing its 686 // body. See Issue 10069. 687 if err == ErrUseLastResponse { 688 return resp, nil 689 } 690 691 // Close the previous response's body. But 692 // read at least some of the body so if it's 693 // small the underlying TCP connection will be 694 // re-used. No need to check for errors: if it 695 // fails, the Transport won't reuse it anyway. 696 const maxBodySlurpSize = 2 << 10 697 if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize { 698 io.CopyN(io.Discard, resp.Body, maxBodySlurpSize) 699 } 700 resp.Body.Close() 701 702 if err != nil { 703 // Special case for Go 1 compatibility: return both the response 704 // and an error if the CheckRedirect function failed. 705 // See https://golang.org/issue/3795 706 // The resp.Body has already been closed. 707 ue := uerr(err) 708 ue.(*url.Error).URL = loc 709 return resp, ue 710 } 711 } 712 713 reqs = append(reqs, req) 714 var err error 715 var didTimeout func() bool 716 if resp, didTimeout, err = c.send(req, deadline); err != nil { 717 // c.send() always closes req.Body 718 reqBodyClosed = true 719 if !deadline.IsZero() && didTimeout() { 720 err = &httpError{ 721 err: err.Error() + " (Client.Timeout exceeded while awaiting headers)", 722 timeout: true, 723 } 724 } 725 return nil, uerr(err) 726 } 727 728 var shouldRedirect bool 729 redirectMethod, shouldRedirect, includeBody = redirectBehavior(req.Method, resp, reqs[0]) 730 if !shouldRedirect { 731 return resp, nil 732 } 733 734 req.closeBody() 735 } 736 } 737 738 // makeHeadersCopier makes a function that copies headers from the 739 // initial Request, ireq. For every redirect, this function must be called 740 // so that it can copy headers into the upcoming Request. 741 func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) { 742 // The headers to copy are from the very initial request. 743 // We use a closured callback to keep a reference to these original headers. 744 var ( 745 ireqhdr = cloneOrMakeHeader(ireq.Header) 746 icookies map[string][]*Cookie 747 ) 748 if c.Jar != nil && ireq.Header.Get("Cookie") != "" { 749 icookies = make(map[string][]*Cookie) 750 for _, c := range ireq.Cookies() { 751 icookies[c.Name] = append(icookies[c.Name], c) 752 } 753 } 754 755 preq := ireq // The previous request 756 return func(req *Request) { 757 // If Jar is present and there was some initial cookies provided 758 // via the request header, then we may need to alter the initial 759 // cookies as we follow redirects since each redirect may end up 760 // modifying a pre-existing cookie. 761 // 762 // Since cookies already set in the request header do not contain 763 // information about the original domain and path, the logic below 764 // assumes any new set cookies override the original cookie 765 // regardless of domain or path. 766 // 767 // See https://golang.org/issue/17494 768 if c.Jar != nil && icookies != nil { 769 var changed bool 770 resp := req.Response // The response that caused the upcoming redirect 771 for _, c := range resp.Cookies() { 772 if _, ok := icookies[c.Name]; ok { 773 delete(icookies, c.Name) 774 changed = true 775 } 776 } 777 if changed { 778 ireqhdr.Del("Cookie") 779 var ss []string 780 for _, cs := range icookies { 781 for _, c := range cs { 782 ss = append(ss, c.Name+"="+c.Value) 783 } 784 } 785 sort.Strings(ss) // Ensure deterministic headers 786 ireqhdr.Set("Cookie", strings.Join(ss, "; ")) 787 } 788 } 789 790 // Copy the initial request's Header values 791 // (at least the safe ones). 792 for k, vv := range ireqhdr { 793 if shouldCopyHeaderOnRedirect(k, preq.URL, req.URL) { 794 req.Header[k] = vv 795 } 796 } 797 798 preq = req // Update previous Request with the current request 799 } 800 } 801 802 func defaultCheckRedirect(req *Request, via []*Request) error { 803 if len(via) >= 10 { 804 return errors.New("stopped after 10 redirects") 805 } 806 return nil 807 } 808 809 // Post issues a POST to the specified URL. 810 // 811 // Caller should close resp.Body when done reading from it. 812 // 813 // If the provided body is an io.Closer, it is closed after the 814 // request. 815 // 816 // Post is a wrapper around DefaultClient.Post. 817 // 818 // To set custom headers, use NewRequest and DefaultClient.Do. 819 // 820 // See the Client.Do method documentation for details on how redirects 821 // are handled. 822 // 823 // To make a request with a specified context.Context, use NewRequestWithContext 824 // and DefaultClient.Do. 825 func Post(url, contentType string, body io.Reader) (resp *Response, err error) { 826 return DefaultClient.Post(url, contentType, body) 827 } 828 829 // Post issues a POST to the specified URL. 830 // 831 // Caller should close resp.Body when done reading from it. 832 // 833 // If the provided body is an io.Closer, it is closed after the 834 // request. 835 // 836 // To set custom headers, use NewRequest and Client.Do. 837 // 838 // To make a request with a specified context.Context, use NewRequestWithContext 839 // and Client.Do. 840 // 841 // See the Client.Do method documentation for details on how redirects 842 // are handled. 843 func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) { 844 req, err := NewRequest("POST", url, body) 845 if err != nil { 846 return nil, err 847 } 848 req.Header.Set("Content-Type", contentType) 849 return c.Do(req) 850 } 851 852 // PostForm issues a POST to the specified URL, with data's keys and 853 // values URL-encoded as the request body. 854 // 855 // The Content-Type header is set to application/x-www-form-urlencoded. 856 // To set other headers, use NewRequest and DefaultClient.Do. 857 // 858 // When err is nil, resp always contains a non-nil resp.Body. 859 // Caller should close resp.Body when done reading from it. 860 // 861 // PostForm is a wrapper around DefaultClient.PostForm. 862 // 863 // See the Client.Do method documentation for details on how redirects 864 // are handled. 865 // 866 // To make a request with a specified context.Context, use NewRequestWithContext 867 // and DefaultClient.Do. 868 func PostForm(url string, data url.Values) (resp *Response, err error) { 869 return DefaultClient.PostForm(url, data) 870 } 871 872 // PostForm issues a POST to the specified URL, 873 // with data's keys and values URL-encoded as the request body. 874 // 875 // The Content-Type header is set to application/x-www-form-urlencoded. 876 // To set other headers, use NewRequest and Client.Do. 877 // 878 // When err is nil, resp always contains a non-nil resp.Body. 879 // Caller should close resp.Body when done reading from it. 880 // 881 // See the Client.Do method documentation for details on how redirects 882 // are handled. 883 // 884 // To make a request with a specified context.Context, use NewRequestWithContext 885 // and Client.Do. 886 func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) { 887 return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) 888 } 889 890 // Head issues a HEAD to the specified URL. If the response is one of 891 // the following redirect codes, Head follows the redirect, up to a 892 // maximum of 10 redirects: 893 // 894 // 301 (Moved Permanently) 895 // 302 (Found) 896 // 303 (See Other) 897 // 307 (Temporary Redirect) 898 // 308 (Permanent Redirect) 899 // 900 // Head is a wrapper around DefaultClient.Head. 901 // 902 // To make a request with a specified context.Context, use NewRequestWithContext 903 // and DefaultClient.Do. 904 func Head(url string) (resp *Response, err error) { 905 return DefaultClient.Head(url) 906 } 907 908 // Head issues a HEAD to the specified URL. If the response is one of the 909 // following redirect codes, Head follows the redirect after calling the 910 // Client's CheckRedirect function: 911 // 912 // 301 (Moved Permanently) 913 // 302 (Found) 914 // 303 (See Other) 915 // 307 (Temporary Redirect) 916 // 308 (Permanent Redirect) 917 // 918 // To make a request with a specified context.Context, use NewRequestWithContext 919 // and Client.Do. 920 func (c *Client) Head(url string) (resp *Response, err error) { 921 req, err := NewRequest("HEAD", url, nil) 922 if err != nil { 923 return nil, err 924 } 925 return c.Do(req) 926 } 927 928 // CloseIdleConnections closes any connections on its Transport which 929 // were previously connected from previous requests but are now 930 // sitting idle in a "keep-alive" state. It does not interrupt any 931 // connections currently in use. 932 // 933 // If the Client's Transport does not have a CloseIdleConnections method 934 // then this method does nothing. 935 func (c *Client) CloseIdleConnections() { 936 type closeIdler interface { 937 CloseIdleConnections() 938 } 939 if tr, ok := c.transport().(closeIdler); ok { 940 tr.CloseIdleConnections() 941 } 942 } 943 944 // cancelTimerBody is an io.ReadCloser that wraps rc with two features: 945 // 1. On Read error or close, the stop func is called. 946 // 2. On Read failure, if reqDidTimeout is true, the error is wrapped and 947 // marked as net.Error that hit its timeout. 948 type cancelTimerBody struct { 949 stop func() // stops the time.Timer waiting to cancel the request 950 rc io.ReadCloser 951 reqDidTimeout func() bool 952 } 953 954 func (b *cancelTimerBody) Read(p []byte) (n int, err error) { 955 n, err = b.rc.Read(p) 956 if err == nil { 957 return n, nil 958 } 959 if err == io.EOF { 960 return n, err 961 } 962 if b.reqDidTimeout() { 963 err = &httpError{ 964 err: err.Error() + " (Client.Timeout or context cancellation while reading body)", 965 timeout: true, 966 } 967 } 968 return n, err 969 } 970 971 func (b *cancelTimerBody) Close() error { 972 err := b.rc.Close() 973 b.stop() 974 return err 975 } 976 977 func shouldCopyHeaderOnRedirect(headerKey string, initial, dest *url.URL) bool { 978 switch CanonicalHeaderKey(headerKey) { 979 case "Authorization", "Www-Authenticate", "Cookie", "Cookie2": 980 // Permit sending auth/cookie headers from "foo.com" 981 // to "sub.foo.com". 982 983 // Note that we don't send all cookies to subdomains 984 // automatically. This function is only used for 985 // Cookies set explicitly on the initial outgoing 986 // client request. Cookies automatically added via the 987 // CookieJar mechanism continue to follow each 988 // cookie's scope as set by Set-Cookie. But for 989 // outgoing requests with the Cookie header set 990 // directly, we don't know their scope, so we assume 991 // it's for *.domain.com. 992 993 ihost := canonicalAddr(initial) 994 dhost := canonicalAddr(dest) 995 return isDomainOrSubdomain(dhost, ihost) 996 } 997 // All other headers are copied: 998 return true 999 } 1000 1001 // isDomainOrSubdomain reports whether sub is a subdomain (or exact 1002 // match) of the parent domain. 1003 // 1004 // Both domains must already be in canonical form. 1005 func isDomainOrSubdomain(sub, parent string) bool { 1006 if sub == parent { 1007 return true 1008 } 1009 // If sub is "foo.example.com" and parent is "example.com", 1010 // that means sub must end in "."+parent. 1011 // Do it without allocating. 1012 if !strings.HasSuffix(sub, parent) { 1013 return false 1014 } 1015 return sub[len(sub)-len(parent)-1] == '.' 1016 } 1017 1018 func stripPassword(u *url.URL) string { 1019 _, passSet := u.User.Password() 1020 if passSet { 1021 return strings.Replace(u.String(), u.User.String()+"@", u.User.Username()+":***@", 1) 1022 } 1023 return u.String() 1024 } 1025