// The retryablehttp package provides a familiar HTTP client interface with // automatic retries and exponential backoff. It is a thin wrapper over the // standard net/http client library and exposes nearly the same public API. // This makes retryablehttp very easy to drop into existing programs. // // retryablehttp performs automatic retries under certain conditions. Mainly, if // an error is returned by the client (connection errors etc), or if a 500-range // response is received, then a retry is invoked. Otherwise, the response is // returned and left to the caller to interpret. // // The main difference from net/http is that requests which take a request body // (POST/PUT et. al) require an io.ReadSeeker to be provided. This enables the // request body to be "rewound" if the initial request fails so that the full // request can be attempted again. package retryablehttp import ( "fmt" "io" "io/ioutil" "log" "math" "net/http" "net/url" "os" "strings" "time" "github.com/hashicorp/go-cleanhttp" ) var ( // Default retry configuration defaultRetryWaitMin = 1 * time.Second defaultRetryWaitMax = 5 * time.Minute defaultRetryMax = 32 // defaultClient is used for performing requests without explicitly making // a new client. It is purposely private to avoid modifications. defaultClient = NewClient() ) // LenReader is an interface implemented by many in-memory io.Reader's. Used // for automatically sending the right Content-Length header when possible. type LenReader interface { Len() int } // Request wraps the metadata needed to create HTTP requests. type Request struct { // body is a seekable reader over the request body payload. This is // used to rewind the request data in between retries. body io.ReadSeeker // Embed an HTTP request directly. This makes a *Request act exactly // like an *http.Request so that all meta methods are supported. *http.Request } // NewRequest creates a new wrapped request. func NewRequest(method, url string, body io.ReadSeeker) (*Request, error) { // Wrap the body in a noop ReadCloser if non-nil. This prevents the // reader from being closed by the HTTP client. var rcBody io.ReadCloser if body != nil { rcBody = ioutil.NopCloser(body) } // Make the request with the noop-closer for the body. httpReq, err := http.NewRequest(method, url, rcBody) if err != nil { return nil, err } // Check if we can set the Content-Length automatically. if lr, ok := body.(LenReader); ok { httpReq.ContentLength = int64(lr.Len()) } return &Request{body, httpReq}, nil } // RequestLogHook allows a function to run before each retry. The HTTP // request which will be made, and the retry number (0 for the initial // request) are available to users. The internal logger is exposed to // consumers. type RequestLogHook func(*log.Logger, *http.Request, int) // ResponseLogHook is like RequestLogHook, but allows running a function // on each HTTP response. This function will be invoked at the end of // every HTTP request executed, regardless of whether a subsequent retry // needs to be performed or not. If the response body is read or closed // from this method, this will affect the response returned from Do(). type ResponseLogHook func(*log.Logger, *http.Response) // Client is used to make HTTP requests. It adds additional functionality // like automatic retries to tolerate minor outages. type Client struct { HTTPClient *http.Client // Internal HTTP client. Logger *log.Logger // Customer logger instance. RetryWaitMin time.Duration // Minimum time to wait RetryWaitMax time.Duration // Maximum time to wait RetryMax int // Maximum number of retries // RequestLogHook allows a user-supplied function to be called // before each retry. RequestLogHook RequestLogHook // ResponseLogHook allows a user-supplied function to be called // with the response from each HTTP request executed. ResponseLogHook ResponseLogHook } // NewClient creates a new Client with default settings. func NewClient() *Client { return &Client{ HTTPClient: cleanhttp.DefaultClient(), Logger: log.New(os.Stderr, "", log.LstdFlags), RetryWaitMin: defaultRetryWaitMin, RetryWaitMax: defaultRetryWaitMax, RetryMax: defaultRetryMax, } } // Do wraps calling an HTTP method with retries. func (c *Client) Do(req *Request) (*http.Response, error) { c.Logger.Printf("[DEBUG] %s %s", req.Method, req.URL) for i := 0; ; i++ { var code int // HTTP response code // Always rewind the request body when non-nil. if req.body != nil { if _, err := req.body.Seek(0, 0); err != nil { return nil, fmt.Errorf("failed to seek body: %v", err) } } if c.RequestLogHook != nil { c.RequestLogHook(c.Logger, req.Request, i) } // Attempt the request resp, err := c.HTTPClient.Do(req.Request) if err != nil { c.Logger.Printf("[ERR] %s %s request failed: %v", req.Method, req.URL, err) goto RETRY } code = resp.StatusCode // Call the response logger function if provided. if c.ResponseLogHook != nil { c.ResponseLogHook(c.Logger, resp) } // Check the response code. We retry on 500-range responses to allow // the server time to recover, as 500's are typically not permanent // errors and may relate to outages on the server side. if code%500 < 100 { resp.Body.Close() goto RETRY } return resp, nil RETRY: remain := c.RetryMax - i if remain == 0 { break } wait := backoff(c.RetryWaitMin, c.RetryWaitMax, i) desc := fmt.Sprintf("%s %s", req.Method, req.URL) if code > 0 { desc = fmt.Sprintf("%s (status: %d)", desc, code) } c.Logger.Printf("[DEBUG] %s: retrying in %s (%d left)", desc, wait, remain) time.Sleep(wait) } // Return an error if we fall out of the retry loop return nil, fmt.Errorf("%s %s giving up after %d attempts", req.Method, req.URL, c.RetryMax+1) } // Get is a shortcut for doing a GET request without making a new client. func Get(url string) (*http.Response, error) { return defaultClient.Get(url) } // Get is a convenience helper for doing simple GET requests. func (c *Client) Get(url string) (*http.Response, error) { req, err := NewRequest("GET", url, nil) if err != nil { return nil, err } return c.Do(req) } // Head is a shortcut for doing a HEAD request without making a new client. func Head(url string) (*http.Response, error) { return defaultClient.Head(url) } // Head is a convenience method for doing simple HEAD requests. func (c *Client) Head(url string) (*http.Response, error) { req, err := NewRequest("HEAD", url, nil) if err != nil { return nil, err } return c.Do(req) } // Post is a shortcut for doing a POST request without making a new client. func Post(url, bodyType string, body io.ReadSeeker) (*http.Response, error) { return defaultClient.Post(url, bodyType, body) } // Post is a convenience method for doing simple POST requests. func (c *Client) Post(url, bodyType string, body io.ReadSeeker) (*http.Response, error) { req, err := NewRequest("POST", url, body) if err != nil { return nil, err } req.Header.Set("Content-Type", bodyType) return c.Do(req) } // PostForm is a shortcut to perform a POST with form data without creating // a new client. func PostForm(url string, data url.Values) (*http.Response, error) { return defaultClient.PostForm(url, data) } // PostForm is a convenience method for doing simple POST operations using // pre-filled url.Values form data. func (c *Client) PostForm(url string, data url.Values) (*http.Response, error) { return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) } // backoff is used to calculate how long to sleep before retrying // after observing failures. It takes the minimum/maximum wait time and // iteration, and returns the duration to wait. func backoff(min, max time.Duration, iter int) time.Duration { mult := math.Pow(2, float64(iter)) * float64(min) sleep := time.Duration(mult) if float64(sleep) != mult || sleep > max { sleep = max } return sleep }