diff options
Diffstat (limited to 'api.go')
| -rw-r--r-- | api.go | 93 |
1 files changed, 62 insertions, 31 deletions
@@ -1,6 +1,6 @@ /* * Minio Go Library for Amazon S3 Compatible Cloud Storage - * (C) 2015, 2016, 2017 Minio, Inc. + * Copyright 2015-2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,10 +19,9 @@ package minio import ( "bytes" + "context" "crypto/md5" "crypto/sha256" - "encoding/base64" - "encoding/hex" "errors" "fmt" "hash" @@ -87,7 +86,7 @@ type Client struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "3.0.0" + libraryVersion = "4.0.5" ) // User Agent should always following the below style. @@ -178,16 +177,29 @@ func (r *lockedRandSource) Seed(seed int64) { r.lk.Unlock() } -// redirectHeaders copies all headers when following a redirect URL. -// This won't be needed anymore from go 1.8 (https://github.com/golang/go/issues/4800) -func redirectHeaders(req *http.Request, via []*http.Request) error { - if len(via) == 0 { - return nil - } - for key, val := range via[0].Header { - req.Header[key] = val - } - return nil +// getRegionFromURL - parse region from URL if present. +func getRegionFromURL(u url.URL) (region string) { + region = "" + if s3utils.IsGoogleEndpoint(u) { + return + } else if s3utils.IsAmazonChinaEndpoint(u) { + // For china specifically we need to set everything to + // cn-north-1 for now, there is no easier way until AWS S3 + // provides a cleaner compatible API across "us-east-1" and + // China region. + return "cn-north-1" + } else if s3utils.IsAmazonGovCloudEndpoint(u) { + // For us-gov specifically we need to set everything to + // us-gov-west-1 for now, there is no easier way until AWS S3 + // provides a cleaner compatible API across "us-east-1" and + // Gov cloud region. + return "us-gov-west-1" + } + parts := s3utils.AmazonS3Host.FindStringSubmatch(u.Host) + if len(parts) > 1 { + region = parts[1] + } + return region } func privateNew(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) { @@ -211,11 +223,13 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re // Instantiate http client and bucket location cache. clnt.httpClient = &http.Client{ - Transport: http.DefaultTransport, - CheckRedirect: redirectHeaders, + Transport: defaultMinioTransport, } // Sets custom region, if region is empty bucket location cache is used automatically. + if region == "" { + region = getRegionFromURL(clnt.endpointURL) + } clnt.region = region // Instantiate bucket location cache. @@ -328,11 +342,11 @@ type requestMetadata struct { expires int64 // Generated by our internal code. - bucketLocation string - contentBody io.Reader - contentLength int64 - contentSHA256Bytes []byte - contentMD5Bytes []byte + bucketLocation string + contentBody io.Reader + contentLength int64 + contentMD5Base64 string // carries base64 encoded md5sum + contentSHA256Hex string // carries hex encoded sha256sum } // dumpHTTP - dump HTTP request and response. @@ -466,9 +480,11 @@ var successStatus = []int{ // executeMethod - instantiates a given method, and retries the // request upon any error up to maxRetries attempts in a binomially // delayed manner using a standard back off algorithm. -func (c Client) executeMethod(method string, metadata requestMetadata) (res *http.Response, err error) { +func (c Client) executeMethod(ctx context.Context, method string, metadata requestMetadata) (res *http.Response, err error) { var isRetryable bool // Indicates if request can be retried. var bodySeeker io.Seeker // Extracted seeker from io.Reader. + var reqRetry = MaxRetry // Indicates how many times we can retry the request + if metadata.contentBody != nil { // Check if body is seekable then it is retryable. bodySeeker, isRetryable = metadata.contentBody.(io.Seeker) @@ -476,6 +492,11 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt case os.Stdin, os.Stdout, os.Stderr: isRetryable = false } + // Retry only when reader is seekable + if !isRetryable { + reqRetry = 1 + } + // Figure out if the body can be closed - if yes // we will definitely close it upon the function // return. @@ -494,7 +515,7 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt // Blank indentifier is kept here on purpose since 'range' without // blank identifiers is only supported since go1.4 // https://golang.org/doc/go1.4#forrange. - for _ = range c.newRetryTimer(MaxRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter, doneCh) { + for range c.newRetryTimer(reqRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter, doneCh) { // Retry executes the following function body if request has an // error until maxRetries have been exhausted, retry attempts are // performed after waiting for a given period of time in a @@ -517,6 +538,8 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt } return nil, err } + // Add context to request + req = req.WithContext(ctx) // Initiate the request. res, err = c.do(req) @@ -562,9 +585,14 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt // Additionally we should only retry if bucketLocation and custom // region is empty. if metadata.bucketLocation == "" && c.region == "" { - if res.StatusCode == http.StatusBadRequest && errResponse.Region != "" { - c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) - continue // Retry. + if errResponse.Code == "AuthorizationHeaderMalformed" || errResponse.Code == "InvalidRegion" { + if metadata.bucketName != "" && errResponse.Region != "" { + // Gather Cached location only if bucketName is present. + if _, cachedLocationError := c.bucketLocCache.Get(metadata.bucketName); cachedLocationError != false { + c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) + continue // Retry. + } + } } } @@ -687,8 +715,8 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R } // set md5Sum for content protection. - if metadata.contentMD5Bytes != nil { - req.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(metadata.contentMD5Bytes)) + if len(metadata.contentMD5Base64) > 0 { + req.Header.Set("Content-Md5", metadata.contentMD5Base64) } // For anonymous requests just return. @@ -700,14 +728,17 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R case signerType.IsV2(): // Add signature version '2' authorization header. req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) - case signerType.IsStreamingV4() && method == "PUT": + case metadata.objectName != "" && method == "PUT" && metadata.customHeader.Get("X-Amz-Copy-Source") == "" && !c.secure: + // Streaming signature is used by default for a PUT object request. Additionally we also + // look if the initialized client is secure, if yes then we don't need to perform + // streaming signature. req = s3signer.StreamingSignV4(req, accessKeyID, secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC()) default: // Set sha256 sum for signature calculation only with signature version '4'. shaHeader := unsignedPayload - if len(metadata.contentSHA256Bytes) > 0 { - shaHeader = hex.EncodeToString(metadata.contentSHA256Bytes) + if metadata.contentSHA256Hex != "" { + shaHeader = metadata.contentSHA256Hex } req.Header.Set("X-Amz-Content-Sha256", shaHeader) |