From 43652cf02b0139f53e70c14f9324371a5a5d35a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Sipma?= Date: Sat, 8 Dec 2018 10:05:57 +0100 Subject: New upstream version 6.0.11 --- Makefile | 12 +- README.md | 26 +- README_zh_CN.md | 1 - api-compose-object.go | 182 +- api-compose-object_test.go | 96 +- api-get-lifecycle.go | 77 + api-get-object-acl.go | 136 + api-get-object-file.go | 13 +- api-get-object.go | 37 +- api-get-options.go | 8 +- api-get-policy.go | 53 +- api-list.go | 27 +- api-notification.go | 14 +- api-presigned.go | 6 +- api-put-bucket.go | 135 +- api-put-object-context.go | 6 - api-put-object-copy.go | 64 +- api-put-object-encrypted.go | 44 - api-put-object-multipart.go | 19 +- api-put-object-streaming.go | 6 +- api-put-object.go | 55 +- api-put-object_test.go | 54 +- api-remove.go | 31 +- api-select.go | 520 +++ api-stat.go | 9 +- api.go | 243 +- api_unit_test.go | 47 +- appveyor.yml | 7 +- bucket-cache.go | 4 +- bucket-cache_test.go | 2 +- bucket-notification.go | 69 +- constants.go | 15 +- core.go | 35 +- core_test.go | 17 +- docs/API.md | 584 ++-- docs/zh_CN/API.md | 44 +- examples/s3/fputencrypted-object.go | 39 +- examples/s3/get-encrypted-object.go | 34 +- examples/s3/getbucketlifecycle.go | 65 + examples/s3/getbucketpolicy.go | 24 +- examples/s3/getobject-client-encryption.go | 66 + examples/s3/getobjectacl.go | 53 + examples/s3/listbucketpolicies.go | 57 - examples/s3/put-encrypted-object.go | 38 +- examples/s3/putobject-client-encryption.go | 75 + examples/s3/putobject-getobject-sse.go | 43 +- examples/s3/removeobjects.go | 6 + examples/s3/selectobject.go | 73 + examples/s3/setbucketlifecycle.go | 50 + examples/s3/setbucketpolicy.go | 12 +- functional_tests.go | 3902 ++++++++++++++-------- pkg/credentials/file_minio_client.go | 18 +- pkg/credentials/iam_aws.go | 54 +- pkg/credentials/iam_aws_test.go | 46 + pkg/encrypt/cbc.go | 294 -- pkg/encrypt/interface.go | 54 - pkg/encrypt/keys.go | 166 - pkg/encrypt/server-side.go | 195 ++ pkg/policy/bucket-policy.go | 45 +- pkg/policy/bucket-policy_test.go | 209 +- pkg/s3signer/request-signature-streaming_test.go | 4 +- pkg/s3signer/request-signature-v2.go | 50 +- pkg/s3signer/request-signature-v4.go | 2 +- pkg/s3signer/request-signature-v4_test.go | 50 + pkg/s3signer/request-signature_test.go | 52 +- pkg/s3signer/utils.go | 9 + pkg/s3signer/utils_test.go | 26 +- pkg/s3utils/utils.go | 86 +- pkg/s3utils/utils_test.go | 104 +- post-policy.go | 22 + retry.go | 46 +- s3-endpoints.go | 5 +- s3-error.go | 2 +- transport.go | 10 +- utils.go | 37 +- utils_test.go | 37 +- 76 files changed, 5500 insertions(+), 3358 deletions(-) create mode 100644 api-get-lifecycle.go create mode 100644 api-get-object-acl.go delete mode 100644 api-put-object-encrypted.go create mode 100644 api-select.go create mode 100644 examples/s3/getbucketlifecycle.go create mode 100644 examples/s3/getobject-client-encryption.go create mode 100644 examples/s3/getobjectacl.go delete mode 100644 examples/s3/listbucketpolicies.go create mode 100644 examples/s3/putobject-client-encryption.go create mode 100644 examples/s3/selectobject.go create mode 100644 examples/s3/setbucketlifecycle.go delete mode 100644 pkg/encrypt/cbc.go delete mode 100644 pkg/encrypt/interface.go delete mode 100644 pkg/encrypt/keys.go create mode 100644 pkg/encrypt/server-side.go create mode 100644 pkg/s3signer/request-signature-v4_test.go diff --git a/Makefile b/Makefile index 8e0dd25..bad81ff 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,12 @@ all: checks checks: - @go get -u github.com/go-ini/ini/... - @go get -u github.com/mitchellh/go-homedir/... - @go get -u github.com/cheggaaa/pb/... - @go get -u github.com/sirupsen/logrus/... - @go get -u github.com/dustin/go-humanize/... + @go get -t ./... @go vet ./... - @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 go test -race -v ./... - @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 go run functional_tests.go + @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... + @go get github.com/dustin/go-humanize/... + @go get github.com/sirupsen/logrus/... + @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go @mkdir -p /tmp/examples && for i in $(echo examples/s3/*); do go build -o /tmp/examples/$(basename ${i:0:-3}) ${i}; done @go get -u github.com/a8m/mark/... @go get -u github.com/minio/cli/... diff --git a/README.md b/README.md index 2dedc1a..ad9d5e6 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ go get -u github.com/minio/minio-go ## Initialize Minio Client Minio client requires the following four parameters specified to connect to an Amazon S3 compatible object storage. -| Parameter | Description| +| Parameter | Description| | :--- | :--- | -| endpoint | URL to object storage service. | +| endpoint | URL to object storage service. | | accessKeyID | Access key is the user ID that uniquely identifies your account. | | secretAccessKey | Secret key is the password to your account. | | secure | Set this value to 'true' to enable secure (HTTPS) access. | @@ -85,8 +85,9 @@ func main() { } else { log.Fatalln(err) } + } else { + log.Printf("Successfully created %s\n", bucketName) } - log.Printf("Successfully created %s\n", bucketName) // Upload the zip file objectName := "golden-oldies.zip" @@ -106,7 +107,7 @@ func main() { ### Run FileUploader ```sh go run file-uploader.go -2016/08/13 17:03:28 Successfully created mymusic +2016/08/13 17:03:28 Successfully created mymusic 2016/08/13 17:03:40 Successfully uploaded golden-oldies.zip of size 16253413 mc ls play/mymusic/ @@ -114,7 +115,7 @@ mc ls play/mymusic/ ``` ## API Reference -The full API Reference is available here. +The full API Reference is available here. * [Complete API Reference](https://docs.minio.io/docs/golang-client-api-reference) @@ -130,7 +131,6 @@ The full API Reference is available here. ### API Reference : Bucket policy Operations * [`SetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketPolicy) * [`GetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketPolicy) -* [`ListBucketPolicies`](https://docs.minio.io/docs/golang-client-api-reference#ListBucketPolicies) ### API Reference : Bucket notification Operations * [`SetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketNotification) @@ -140,7 +140,7 @@ The full API Reference is available here. ### API Reference : File Object Operations * [`FPutObject`](https://docs.minio.io/docs/golang-client-api-reference#FPutObject) -* [`FGetObject`](https://docs.minio.io/docs/golang-client-api-reference#FPutObject) +* [`FGetObject`](https://docs.minio.io/docs/golang-client-api-reference#FGetObject) * [`FPutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FPutObjectWithContext) * [`FGetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FGetObjectWithContext) @@ -155,10 +155,8 @@ The full API Reference is available here. * [`RemoveObject`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObject) * [`RemoveObjects`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObjects) * [`RemoveIncompleteUpload`](https://docs.minio.io/docs/golang-client-api-reference#RemoveIncompleteUpload) +* [`SelectObjectContent`](https://docs.minio.io/docs/golang-client-api-reference#SelectObjectContent) -### API Reference: Encrypted Object Operations -* [`GetEncryptedObject`](https://docs.minio.io/docs/golang-client-api-reference#GetEncryptedObject) -* [`PutEncryptedObject`](https://docs.minio.io/docs/golang-client-api-reference#PutEncryptedObject) ### API Reference : Presigned Operations * [`PresignedGetObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedGetObject) @@ -187,7 +185,11 @@ The full API Reference is available here. * [setbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketpolicy.go) * [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go) * [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go) - + +### Full Examples : Bucket lifecycle Operations +* [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go) +* [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go) + ### Full Examples : Bucket notification Operations * [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) @@ -224,7 +226,7 @@ The full API Reference is available here. ## Explore Further * [Complete Documentation](https://docs.minio.io) -* [Minio Go Client SDK API Reference](https://docs.minio.io/docs/golang-client-api-reference) +* [Minio Go Client SDK API Reference](https://docs.minio.io/docs/golang-client-api-reference) * [Go Music Player App Full Application Example](https://docs.minio.io/docs/go-music-player-app) ## Contribute diff --git a/README_zh_CN.md b/README_zh_CN.md index 5584f42..a5acf19 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -141,7 +141,6 @@ mc ls play/mymusic/ ### API文档 : 存储桶策略 * [`SetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketPolicy) * [`GetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketPolicy) -* [`ListBucketPolicies`](https://docs.minio.io/docs/golang-client-api-reference#ListBucketPolicies) ### API文档 : 存储桶通知 * [`SetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketNotification) diff --git a/api-compose-object.go b/api-compose-object.go index 81314e3..3ac36c5 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -1,6 +1,6 @@ /* * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * Copyright 2017, 2018 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,66 +19,24 @@ package minio import ( "context" - "encoding/base64" "fmt" + "io" + "io/ioutil" "net/http" "net/url" "strconv" "strings" "time" + "github.com/minio/minio-go/pkg/encrypt" "github.com/minio/minio-go/pkg/s3utils" ) -// SSEInfo - represents Server-Side-Encryption parameters specified by -// a user. -type SSEInfo struct { - key []byte - algo string -} - -// NewSSEInfo - specifies (binary or un-encoded) encryption key and -// algorithm name. If algo is empty, it defaults to "AES256". Ref: -// https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html -func NewSSEInfo(key []byte, algo string) SSEInfo { - if algo == "" { - algo = "AES256" - } - return SSEInfo{key, algo} -} - -// internal method that computes SSE-C headers -func (s *SSEInfo) getSSEHeaders(isCopySource bool) map[string]string { - if s == nil { - return nil - } - - cs := "" - if isCopySource { - cs = "copy-source-" - } - return map[string]string{ - "x-amz-" + cs + "server-side-encryption-customer-algorithm": s.algo, - "x-amz-" + cs + "server-side-encryption-customer-key": base64.StdEncoding.EncodeToString(s.key), - "x-amz-" + cs + "server-side-encryption-customer-key-MD5": sumMD5Base64(s.key), - } -} - -// GetSSEHeaders - computes and returns headers for SSE-C as key-value -// pairs. They can be set as metadata in PutObject* requests (for -// encryption) or be set as request headers in `Core.GetObject` (for -// decryption). -func (s *SSEInfo) GetSSEHeaders() map[string]string { - return s.getSSEHeaders(false) -} - // DestinationInfo - type with information about the object to be // created via server-side copy requests, using the Compose API. type DestinationInfo struct { bucket, object string - - // key for encrypting destination - encryption *SSEInfo + encryption encrypt.ServerSide // if no user-metadata is provided, it is copied from source // (when there is only once source object in the compose @@ -97,9 +55,7 @@ type DestinationInfo struct { // if needed. If nil is passed, and if only a single source (of any // size) is provided in the ComposeObject call, then metadata from the // source is copied to the destination. -func NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, - userMeta map[string]string) (d DestinationInfo, err error) { - +func NewDestinationInfo(bucket, object string, sse encrypt.ServerSide, userMeta map[string]string) (d DestinationInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucket); err != nil { return d, err @@ -125,7 +81,7 @@ func NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, return DestinationInfo{ bucket: bucket, object: object, - encryption: encryptSSEC, + encryption: sse, userMetadata: m, }, nil } @@ -145,7 +101,11 @@ func (d *DestinationInfo) getUserMetaHeadersMap(withCopyDirectiveHeader bool) ma r["x-amz-metadata-directive"] = "REPLACE" } for k, v := range d.userMetadata { - r["x-amz-meta-"+k] = v + if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) { + r[k] = v + } else { + r["x-amz-meta-"+k] = v + } } return r } @@ -154,10 +114,8 @@ func (d *DestinationInfo) getUserMetaHeadersMap(withCopyDirectiveHeader bool) ma // server-side copying APIs. type SourceInfo struct { bucket, object string - - start, end int64 - - decryptKey *SSEInfo + start, end int64 + encryption encrypt.ServerSide // Headers to send with the upload-part-copy request involving // this source object. Headers http.Header @@ -169,23 +127,17 @@ type SourceInfo struct { // `decryptSSEC` is the decryption key using server-side-encryption // with customer provided key. It may be nil if the source is not // encrypted. -func NewSourceInfo(bucket, object string, decryptSSEC *SSEInfo) SourceInfo { +func NewSourceInfo(bucket, object string, sse encrypt.ServerSide) SourceInfo { r := SourceInfo{ bucket: bucket, object: object, start: -1, // range is unspecified by default - decryptKey: decryptSSEC, + encryption: sse, Headers: make(http.Header), } // Set the source header r.Headers.Set("x-amz-copy-source", s3utils.EncodePath(bucket+"/"+object)) - - // Assemble decryption headers for upload-part-copy request - for k, v := range decryptSSEC.getSSEHeaders(true) { - r.Headers.Set(k, v) - } - return r } @@ -245,10 +197,7 @@ func (s *SourceInfo) getProps(c Client) (size int64, etag string, userMeta map[s // Get object info - need size and etag here. Also, decryption // headers are added to the stat request if given. var objInfo ObjectInfo - opts := StatObjectOptions{} - for k, v := range s.decryptKey.getSSEHeaders(false) { - opts.Set(k, v) - } + opts := StatObjectOptions{GetObjectOptions{ServerSideEncryption: encrypt.SSE(s.encryption)}} objInfo, err = c.statObject(context.Background(), s.bucket, s.object, opts) if err != nil { err = ErrInvalidArgument(fmt.Sprintf("Could not stat object - %s/%s: %v", s.bucket, s.object, err)) @@ -400,11 +349,12 @@ func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID str return p, nil } -// ComposeObject - creates an object using server-side copying of +// ComposeObjectWithProgress - creates an object using server-side copying of // existing objects. It takes a list of source objects (with optional // offsets) and concatenates them into a new object using only -// server-side copying operations. -func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error { +// server-side copying operations. Optionally takes progress reader hook +// for applications to look at current progress. +func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error { if len(srcs) < 1 || len(srcs) > maxPartsCount { return ErrInvalidArgument("There must be as least one and up to 10000 source objects.") } @@ -412,10 +362,10 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error { srcSizes := make([]int64, len(srcs)) var totalSize, size, totalParts int64 var srcUserMeta map[string]string - var etag string + etags := make([]string, len(srcs)) var err error for i, src := range srcs { - size, etag, srcUserMeta, err = src.getProps(c) + size, etags[i], srcUserMeta, err = src.getProps(c) if err != nil { return err } @@ -427,15 +377,6 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error { fmt.Sprintf("Client side encryption is used in source object %s/%s", src.bucket, src.object)) } - // Since we did a HEAD to get size, we use the ETag - // value to make sure the object has not changed by - // the time we perform the copy. This is done, only if - // the user has not set their own ETag match - // condition. - if src.Headers.Get("x-amz-copy-source-if-match") == "" { - src.SetMatchETagCond(etag) - } - // Check if a segment is specified, and if so, is the // segment within object bounds? if src.start != -1 { @@ -476,44 +417,22 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error { // Single source object case (i.e. when only one source is // involved, it is being copied wholly and at most 5GiB in - // size). - if totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize { - h := srcs[0].Headers - // Add destination encryption headers - for k, v := range dst.encryption.getSSEHeaders(false) { - h.Set(k, v) - } + // size, emptyfiles are also supported). + if (totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize) || (totalSize == 0) { + return c.CopyObjectWithProgress(dst, srcs[0], progress) + } - // If no user metadata is specified (and so, the - // for-loop below is not entered), metadata from the - // source is copied to the destination (due to - // single-part copy-object PUT request behaviour). - for k, v := range dst.getUserMetaHeadersMap(true) { - h.Set(k, v) - } + // Now, handle multipart-copy cases. - // Send copy request - resp, err := c.executeMethod(ctx, "PUT", requestMetadata{ - bucketName: dst.bucket, - objectName: dst.object, - customHeader: h, - }) - defer closeResponse(resp) - if err != nil { - return err - } - // Check if we got an error response. - if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, dst.bucket, dst.object) + // 1. Ensure that the object has not been changed while + // we are copying data. + for i, src := range srcs { + if src.Headers.Get("x-amz-copy-source-if-match") == "" { + src.SetMatchETagCond(etags[i]) } - - // Return nil on success. - return nil } - // Now, handle multipart-copy cases. - - // 1. Initiate a new multipart upload. + // 2. Initiate a new multipart upload. // Set user-metadata on the destination object. If no // user-metadata is specified, and there is only one source, @@ -527,19 +446,23 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error { for k, v := range metaMap { metaHeaders[k] = v } - uploadID, err := c.newUploadID(ctx, dst.bucket, dst.object, PutObjectOptions{UserMetadata: metaHeaders}) + + uploadID, err := c.newUploadID(ctx, dst.bucket, dst.object, PutObjectOptions{ServerSideEncryption: dst.encryption, UserMetadata: metaHeaders}) if err != nil { return err } - // 2. Perform copy part uploads + // 3. Perform copy part uploads objParts := []CompletePart{} partIndex := 1 for i, src := range srcs { h := src.Headers + if src.encryption != nil { + encrypt.SSECopy(src.encryption).Marshal(h) + } // Add destination encryption headers - for k, v := range dst.encryption.getSSEHeaders(false) { - h.Set(k, v) + if dst.encryption != nil { + dst.encryption.Marshal(h) } // calculate start/end indices of parts after @@ -559,12 +482,15 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error { if err != nil { return err } + if progress != nil { + io.CopyN(ioutil.Discard, progress, end-start+1) + } objParts = append(objParts, complPart) partIndex++ } } - // 3. Make final complete-multipart request. + // 4. Make final complete-multipart request. _, err = c.completeMultipartUpload(ctx, dst.bucket, dst.object, uploadID, completeMultipartUpload{Parts: objParts}) if err != nil { @@ -573,10 +499,20 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error { return nil } -// partsRequired is ceiling(size / copyPartSize) +// ComposeObject - creates an object using server-side copying of +// existing objects. It takes a list of source objects (with optional +// offsets) and concatenates them into a new object using only +// server-side copying operations. +func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error { + return c.ComposeObjectWithProgress(dst, srcs, nil) +} + +// partsRequired is maximum parts possible with +// max part size of ceiling(maxMultipartPutObjectSize / (maxPartsCount - 1)) func partsRequired(size int64) int64 { - r := size / copyPartSize - if size%copyPartSize > 0 { + maxPartSize := maxMultipartPutObjectSize / (maxPartsCount - 1) + r := size / int64(maxPartSize) + if size%int64(maxPartSize) > 0 { r++ } return r diff --git a/api-compose-object_test.go b/api-compose-object_test.go index 0f22a96..295bbc2 100644 --- a/api-compose-object_test.go +++ b/api-compose-object_test.go @@ -18,6 +18,7 @@ package minio import ( "reflect" + "strings" "testing" ) @@ -35,16 +36,19 @@ func TestPartsRequired(t *testing.T) { }{ {0, 0}, {1, 1}, - {gb5, 1}, - {2 * gb5, 2}, - {gb10p1, 3}, - {gb10p2, 3}, + {gb5, 10}, + {gb5p1, 10}, + {2 * gb5, 20}, + {gb10p1, 20}, + {gb10p2, 20}, + {gb10p1 + gb10p2, 40}, + {maxMultipartPutObjectSize, 10000}, } for i, testCase := range testCases { res := partsRequired(testCase.size) if res != testCase.ref { - t.Errorf("Test %d - output did not match with reference results", i+1) + t.Errorf("Test %d - output did not match with reference results, Expected %d, got %d", i+1, testCase.ref, res) } } } @@ -63,27 +67,89 @@ func TestCalculateEvenSplits(t *testing.T) { {1, SourceInfo{start: -1}, []int64{0}, []int64{0}}, {1, SourceInfo{start: 0}, []int64{0}, []int64{0}}, - {gb1, SourceInfo{start: -1}, []int64{0}, []int64{gb1 - 1}}, - {gb5, SourceInfo{start: -1}, []int64{0}, []int64{gb5 - 1}}, + {gb1, SourceInfo{start: -1}, []int64{0, 536870912}, []int64{536870911, 1073741823}}, + {gb5, SourceInfo{start: -1}, + []int64{0, 536870912, 1073741824, 1610612736, 2147483648, 2684354560, + 3221225472, 3758096384, 4294967296, 4831838208}, + []int64{536870911, 1073741823, 1610612735, 2147483647, 2684354559, 3221225471, + 3758096383, 4294967295, 4831838207, 5368709119}, + }, // 2 part splits - {gb5p1, SourceInfo{start: -1}, []int64{0, gb5/2 + 1}, []int64{gb5 / 2, gb5}}, - {gb5p1, SourceInfo{start: -1}, []int64{0, gb5/2 + 1}, []int64{gb5 / 2, gb5}}, + {gb5p1, SourceInfo{start: -1}, + []int64{0, 536870913, 1073741825, 1610612737, 2147483649, 2684354561, + 3221225473, 3758096385, 4294967297, 4831838209}, + []int64{536870912, 1073741824, 1610612736, 2147483648, 2684354560, 3221225472, + 3758096384, 4294967296, 4831838208, 5368709120}, + }, + {gb5p1, SourceInfo{start: -1}, + []int64{0, 536870913, 1073741825, 1610612737, 2147483649, 2684354561, + 3221225473, 3758096385, 4294967297, 4831838209}, + []int64{536870912, 1073741824, 1610612736, 2147483648, 2684354560, 3221225472, + 3758096384, 4294967296, 4831838208, 5368709120}, + }, // 3 part splits {gb10p1, SourceInfo{start: -1}, - []int64{0, gb10p1/3 + 1, 2*gb10p1/3 + 1}, - []int64{gb10p1 / 3, 2 * gb10p1 / 3, gb10p1 - 1}}, - + []int64{0, 536870913, 1073741825, 1610612737, 2147483649, 2684354561, + 3221225473, 3758096385, 4294967297, 4831838209, 5368709121, + 5905580033, 6442450945, 6979321857, 7516192769, 8053063681, + 8589934593, 9126805505, 9663676417, 10200547329}, + []int64{536870912, 1073741824, 1610612736, 2147483648, 2684354560, + 3221225472, 3758096384, 4294967296, 4831838208, 5368709120, + 5905580032, 6442450944, 6979321856, 7516192768, 8053063680, + 8589934592, 9126805504, 9663676416, 10200547328, 10737418240}, + }, {gb10p2, SourceInfo{start: -1}, - []int64{0, gb10p2 / 3, 2 * gb10p2 / 3}, - []int64{gb10p2/3 - 1, 2*gb10p2/3 - 1, gb10p2 - 1}}, + []int64{0, 536870913, 1073741826, 1610612738, 2147483650, 2684354562, + 3221225474, 3758096386, 4294967298, 4831838210, 5368709122, + 5905580034, 6442450946, 6979321858, 7516192770, 8053063682, + 8589934594, 9126805506, 9663676418, 10200547330}, + []int64{536870912, 1073741825, 1610612737, 2147483649, 2684354561, + 3221225473, 3758096385, 4294967297, 4831838209, 5368709121, + 5905580033, 6442450945, 6979321857, 7516192769, 8053063681, + 8589934593, 9126805505, 9663676417, 10200547329, 10737418241}, + }, } for i, testCase := range testCases { resStart, resEnd := calculateEvenSplits(testCase.size, testCase.src) if !reflect.DeepEqual(testCase.starts, resStart) || !reflect.DeepEqual(testCase.ends, resEnd) { - t.Errorf("Test %d - output did not match with reference results", i+1) + t.Errorf("Test %d - output did not match with reference results, Expected %d/%d, got %d/%d", i+1, testCase.starts, testCase.ends, resStart, resEnd) + } + } +} + +func TestGetUserMetaHeadersMap(t *testing.T) { + + userMetadata := map[string]string{ + "test": "test", + "x-amz-acl": "public-read-write", + "content-type": "application/binary", + "X-Amz-Storage-Class": "rrs", + "x-amz-grant-write": "test@exo.ch", + } + + destInfo := &DestinationInfo{"bucket", "object", nil, userMetadata} + + r := destInfo.getUserMetaHeadersMap(true) + + i := 0 + + if _, ok := r["x-amz-metadata-directive"]; !ok { + t.Errorf("Test %d - metadata directive was expected but is missing", i) + i++ + } + + for k := range r { + if strings.HasSuffix(k, "test") && !strings.HasPrefix(k, "x-amz-meta-") { + t.Errorf("Test %d - meta %q was expected as an x amz meta", i, k) + i++ + } + + if !strings.HasSuffix(k, "test") && strings.HasPrefix(k, "x-amz-meta-") { + t.Errorf("Test %d - an amz/standard/storageClass Header was expected but got an x amz meta data", i) + i++ } } } diff --git a/api-get-lifecycle.go b/api-get-lifecycle.go new file mode 100644 index 0000000..8097bfc --- /dev/null +++ b/api-get-lifecycle.go @@ -0,0 +1,77 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "io/ioutil" + "net/http" + "net/url" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// GetBucketLifecycle - get bucket lifecycle. +func (c Client) GetBucketLifecycle(bucketName string) (string, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return "", err + } + bucketLifecycle, err := c.getBucketLifecycle(bucketName) + if err != nil { + errResponse := ToErrorResponse(err) + if errResponse.Code == "NoSuchLifecycleConfiguration" { + return "", nil + } + return "", err + } + return bucketLifecycle, nil +} + +// Request server for current bucket lifecycle. +func (c Client) getBucketLifecycle(bucketName string) (string, error) { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("lifecycle", "") + + // Execute GET on bucket to get lifecycle. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return "", err + } + + if resp != nil { + if resp.StatusCode != http.StatusOK { + return "", httpRespToErrorResponse(resp, bucketName, "") + } + } + + bucketLifecycleBuf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + lifecycle := string(bucketLifecycleBuf) + return lifecycle, err +} diff --git a/api-get-object-acl.go b/api-get-object-acl.go new file mode 100644 index 0000000..af5544d --- /dev/null +++ b/api-get-object-acl.go @@ -0,0 +1,136 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "net/http" + "net/url" +) + +type accessControlPolicy struct { + Owner struct { + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` + } `xml:"Owner"` + AccessControlList struct { + Grant []struct { + Grantee struct { + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` + URI string `xml:"URI"` + } `xml:"Grantee"` + Permission string `xml:"Permission"` + } `xml:"Grant"` + } `xml:"AccessControlList"` +} + +//GetObjectACL get object ACLs +func (c Client) GetObjectACL(bucketName, objectName string) (*ObjectInfo, error) { + + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: url.Values{ + "acl": []string{""}, + }, + }) + if err != nil { + return nil, err + } + defer closeResponse(resp) + + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, objectName) + } + + res := &accessControlPolicy{} + + if err := xmlDecoder(resp.Body, res); err != nil { + return nil, err + } + + objInfo, err := c.statObject(context.Background(), bucketName, objectName, StatObjectOptions{}) + if err != nil { + return nil, err + } + + cannedACL := getCannedACL(res) + if cannedACL != "" { + objInfo.Metadata.Add("X-Amz-Acl", cannedACL) + return &objInfo, nil + } + + grantACL := getAmzGrantACL(res) + for k, v := range grantACL { + objInfo.Metadata[k] = v + } + + return &objInfo, nil +} + +func getCannedACL(aCPolicy *accessControlPolicy) string { + grants := aCPolicy.AccessControlList.Grant + + switch { + case len(grants) == 1: + if grants[0].Grantee.URI == "" && grants[0].Permission == "FULL_CONTROL" { + return "private" + } + case len(grants) == 2: + for _, g := range grants { + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && g.Permission == "READ" { + return "authenticated-read" + } + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "READ" { + return "public-read" + } + if g.Permission == "READ" && g.Grantee.ID == aCPolicy.Owner.ID { + return "bucket-owner-read" + } + } + case len(grants) == 3: + for _, g := range grants { + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "WRITE" { + return "public-read-write" + } + } + } + return "" +} + +func getAmzGrantACL(aCPolicy *accessControlPolicy) map[string][]string { + grants := aCPolicy.AccessControlList.Grant + res := map[string][]string{} + + for _, g := range grants { + switch { + case g.Permission == "READ": + res["X-Amz-Grant-Read"] = append(res["X-Amz-Grant-Read"], "id="+g.Grantee.ID) + case g.Permission == "WRITE": + res["X-Amz-Grant-Write"] = append(res["X-Amz-Grant-Write"], "id="+g.Grantee.ID) + case g.Permission == "READ_ACP": + res["X-Amz-Grant-Read-Acp"] = append(res["X-Amz-Grant-Read-Acp"], "id="+g.Grantee.ID) + case g.Permission == "WRITE_ACP": + res["X-Amz-Grant-Write-Acp"] = append(res["X-Amz-Grant-Write-Acp"], "id="+g.Grantee.ID) + case g.Permission == "FULL_CONTROL": + res["X-Amz-Grant-Full-Control"] = append(res["X-Amz-Grant-Full-Control"], "id="+g.Grantee.ID) + } + } + return res +} diff --git a/api-get-object-file.go b/api-get-object-file.go index 2b58220..a852220 100644 --- a/api-get-object-file.go +++ b/api-get-object-file.go @@ -18,14 +18,11 @@ package minio import ( + "context" "io" "os" "path/filepath" - "github.com/minio/minio-go/pkg/encrypt" - - "context" - "github.com/minio/minio-go/pkg/s3utils" ) @@ -40,14 +37,6 @@ func (c Client) FGetObject(bucketName, objectName, filePath string, opts GetObje return c.fGetObjectWithContext(context.Background(), bucketName, objectName, filePath, opts) } -// FGetEncryptedObject - Decrypt and store an object at filePath. -func (c Client) FGetEncryptedObject(bucketName, objectName, filePath string, materials encrypt.Materials) error { - if materials == nil { - return ErrInvalidArgument("Unable to recognize empty encryption properties") - } - return c.FGetObject(bucketName, objectName, filePath, GetObjectOptions{Materials: materials}) -} - // fGetObjectWithContext - fgetObject wrapper function with context func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error { // Input validation. diff --git a/api-get-object.go b/api-get-object.go index 50bbc22..0bf556e 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -27,20 +27,9 @@ import ( "sync" "time" - "github.com/minio/minio-go/pkg/encrypt" "github.com/minio/minio-go/pkg/s3utils" ) -// GetEncryptedObject deciphers and streams data stored in the server after applying a specified encryption materials, -// returned stream should be closed by the caller. -func (c Client) GetEncryptedObject(bucketName, objectName string, encryptMaterials encrypt.Materials) (io.ReadCloser, error) { - if encryptMaterials == nil { - return nil, ErrInvalidArgument("Unable to recognize empty encryption properties") - } - - return c.GetObject(bucketName, objectName, GetObjectOptions{Materials: encryptMaterials}) -} - // GetObject - returns an seekable, readable object. func (c Client) GetObject(bucketName, objectName string, opts GetObjectOptions) (*Object, error) { return c.getObjectWithContext(context.Background(), bucketName, objectName, opts) @@ -127,6 +116,9 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName } else { // First request is a Stat or Seek call. // Only need to run a StatObject until an actual Read or ReadAt request comes through. + + // Remove range header if already set, for stat Operations to get original file size. + delete(opts.headers, "Range") objectInfo, err = c.statObject(ctx, bucketName, objectName, StatObjectOptions{opts}) if err != nil { resCh <- getResponse{ @@ -142,6 +134,8 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName } } } else if req.settingObjectInfo { // Request is just to get objectInfo. + // Remove range header if already set, for stat Operations to get original file size. + delete(opts.headers, "Range") if etag != "" { opts.SetMatchETag(etag) } @@ -381,13 +375,11 @@ func (o *Object) Stat() (ObjectInfo, error) { // This is the first request. if !o.isStarted || !o.objectInfoSet { - statReq := getRequest{ + // Send the request and get the response. + _, err := o.doGetRequest(getRequest{ isFirstReq: !o.isStarted, settingObjectInfo: !o.objectInfoSet, - } - - // Send the request and get the response. - _, err := o.doGetRequest(statReq) + }) if err != nil { o.prevErr = err return ObjectInfo{}, err @@ -493,7 +485,7 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { // Negative offset is valid for whence of '2'. if offset < 0 && whence != 2 { - return 0, ErrInvalidArgument(fmt.Sprintf("Negative position not allowed for %d.", whence)) + return 0, ErrInvalidArgument(fmt.Sprintf("Negative position not allowed for %d", whence)) } // This is the first request. So before anything else @@ -662,15 +654,6 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op Metadata: extractObjMetadata(resp.Header), } - reader := resp.Body - if opts.Materials != nil { - err = opts.Materials.SetupDecryptMode(reader, objectStat.Metadata.Get(amzHeaderIV), objectStat.Metadata.Get(amzHeaderKey)) - if err != nil { - return nil, ObjectInfo{}, err - } - reader = opts.Materials - } - // do not close body here, caller will close - return reader, objectStat, nil + return resp.Body, objectStat, nil } diff --git a/api-get-options.go b/api-get-options.go index dd70415..dbf062d 100644 --- a/api-get-options.go +++ b/api-get-options.go @@ -28,9 +28,8 @@ import ( // GetObjectOptions are used to specify additional headers or options // during GET requests. type GetObjectOptions struct { - headers map[string]string - - Materials encrypt.Materials + headers map[string]string + ServerSideEncryption encrypt.ServerSide } // StatObjectOptions are used to specify additional headers or options @@ -45,6 +44,9 @@ func (o GetObjectOptions) Header() http.Header { for k, v := range o.headers { headers.Set(k, v) } + if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() == encrypt.SSEC { + o.ServerSideEncryption.Marshal(headers) + } return headers } diff --git a/api-get-policy.go b/api-get-policy.go index a4259c9..12d4c59 100644 --- a/api-get-policy.go +++ b/api-get-policy.go @@ -19,62 +19,32 @@ package minio import ( "context" - "encoding/json" "io/ioutil" "net/http" "net/url" - "github.com/minio/minio-go/pkg/policy" "github.com/minio/minio-go/pkg/s3utils" ) // GetBucketPolicy - get bucket policy at a given path. -func (c Client) GetBucketPolicy(bucketName, objectPrefix string) (bucketPolicy policy.BucketPolicy, err error) { +func (c Client) GetBucketPolicy(bucketName string) (string, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return policy.BucketPolicyNone, err + return "", err } - if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { - return policy.BucketPolicyNone, err - } - policyInfo, err := c.getBucketPolicy(bucketName) - if err != nil { - errResponse := ToErrorResponse(err) - if errResponse.Code == "NoSuchBucketPolicy" { - return policy.BucketPolicyNone, nil - } - return policy.BucketPolicyNone, err - } - return policy.GetPolicy(policyInfo.Statements, bucketName, objectPrefix), nil -} - -// ListBucketPolicies - list all policies for a given prefix and all its children. -func (c Client) ListBucketPolicies(bucketName, objectPrefix string) (bucketPolicies map[string]policy.BucketPolicy, err error) { - // Input validation. - if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return map[string]policy.BucketPolicy{}, err - } - if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { - return map[string]policy.BucketPolicy{}, err - } - policyInfo, err := c.getBucketPolicy(bucketName) + bucketPolicy, err := c.getBucketPolicy(bucketName) if err != nil { errResponse := ToErrorResponse(err) if errResponse.Code == "NoSuchBucketPolicy" { - return map[string]policy.BucketPolicy{}, nil + return "", nil } - return map[string]policy.BucketPolicy{}, err + return "", err } - return policy.GetPolicies(policyInfo.Statements, bucketName), nil -} - -// Default empty bucket access policy. -var emptyBucketAccessPolicy = policy.BucketAccessPolicy{ - Version: "2012-10-17", + return bucketPolicy, nil } // Request server for current bucket policy. -func (c Client) getBucketPolicy(bucketName string) (policy.BucketAccessPolicy, error) { +func (c Client) getBucketPolicy(bucketName string) (string, error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) @@ -89,21 +59,20 @@ func (c Client) getBucketPolicy(bucketName string) (policy.BucketAccessPolicy, e defer closeResponse(resp) if err != nil { - return emptyBucketAccessPolicy, err + return "", err } if resp != nil { if resp.StatusCode != http.StatusOK { - return emptyBucketAccessPolicy, httpRespToErrorResponse(resp, bucketName, "") + return "", httpRespToErrorResponse(resp, bucketName, "") } } bucketPolicyBuf, err := ioutil.ReadAll(resp.Body) if err != nil { - return emptyBucketAccessPolicy, err + return "", err } - policy := policy.BucketAccessPolicy{} - err = json.Unmarshal(bucketPolicyBuf, &policy) + policy := string(bucketPolicyBuf) return policy, err } diff --git a/api-list.go b/api-list.go index 3cfb47d..04f7573 100644 --- a/api-list.go +++ b/api-list.go @@ -118,7 +118,7 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d var continuationToken string for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, 1000) + result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, 1000, "") if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -171,11 +171,12 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // You can use the request parameters as selection criteria to return a subset of the objects in a bucket. // request parameters :- // --------- -// ?continuation-token - Specifies the key to start with when listing objects in a bucket. +// ?continuation-token - Used to continue iterating over a set of objects // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. -func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) { +// ?start-after - Specifies the key to start after when listing objects in a bucket. +func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketV2Result{}, err @@ -216,6 +217,11 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s // Set max keys. urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + // Set start-after + if startAfter != "" { + urlValues.Set("start-after", startAfter) + } + // Execute GET on bucket to list objects. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ bucketName: bucketName, @@ -627,30 +633,27 @@ func (c Client) listObjectParts(bucketName, objectName, uploadID string) (partsI return partsInfo, nil } -// findUploadID lists all incomplete uploads and finds the uploadID of the matching object name. -func (c Client) findUploadID(bucketName, objectName string) (uploadID string, err error) { +// findUploadIDs lists all incomplete uploads and find the uploadIDs of the matching object name. +func (c Client) findUploadIDs(bucketName, objectName string) ([]string, error) { + var uploadIDs []string // Make list incomplete uploads recursive. isRecursive := true // Turn off size aggregation of individual parts, in this request. isAggregateSize := false - // latestUpload to track the latest multipart info for objectName. - var latestUpload ObjectMultipartInfo // Create done channel to cleanup the routine. doneCh := make(chan struct{}) defer close(doneCh) // List all incomplete uploads. for mpUpload := range c.listIncompleteUploads(bucketName, objectName, isRecursive, isAggregateSize, doneCh) { if mpUpload.Err != nil { - return "", mpUpload.Err + return nil, mpUpload.Err } if objectName == mpUpload.Key { - if mpUpload.Initiated.Sub(latestUpload.Initiated) > 0 { - latestUpload = mpUpload - } + uploadIDs = append(uploadIDs, mpUpload.UploadID) } } // Return the latest upload id. - return latestUpload.UploadID, nil + return uploadIDs, nil } // getTotalMultipartSize - calculate total uploaded size for the a given multipart object. diff --git a/api-notification.go b/api-notification.go index 3f5b30a..1c01e36 100644 --- a/api-notification.go +++ b/api-notification.go @@ -150,7 +150,7 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even } // Check ARN partition to verify if listening bucket is supported - if s3utils.IsAmazonEndpoint(c.endpointURL) || s3utils.IsGoogleEndpoint(c.endpointURL) { + if s3utils.IsAmazonEndpoint(*c.endpointURL) || s3utils.IsGoogleEndpoint(*c.endpointURL) { notificationInfoCh <- NotificationInfo{ Err: ErrAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"), } @@ -205,13 +205,11 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even if err = json.Unmarshal(bio.Bytes(), ¬ificationInfo); err != nil { continue } - // Send notifications on channel only if there are events received. - if len(notificationInfo.Records) > 0 { - select { - case notificationInfoCh <- notificationInfo: - case <-doneCh: - return - } + // Send notificationInfo + select { + case notificationInfoCh <- notificationInfo: + case <-doneCh: + return } } // Look for any underlying errors. diff --git a/api-presigned.go b/api-presigned.go index 123ad44..a2c0607 100644 --- a/api-presigned.go +++ b/api-presigned.go @@ -119,7 +119,9 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str return nil, nil, err } - u, err = c.makeTargetURL(bucketName, "", location, nil) + isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, bucketName) + + u, err = c.makeTargetURL(bucketName, "", location, isVirtualHost, nil) if err != nil { return nil, nil, err } @@ -148,7 +150,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str policyBase64 := p.base64() p.formData["policy"] = policyBase64 // For Google endpoint set this value to be 'GoogleAccessId'. - if s3utils.IsGoogleEndpoint(c.endpointURL) { + if s3utils.IsGoogleEndpoint(*c.endpointURL) { p.formData["GoogleAccessId"] = accessKeyID } else { // For all other endpoints set this value to be 'AWSAccessKeyId'. diff --git a/api-put-bucket.go b/api-put-bucket.go index bb583a7..33dc0cf 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -20,13 +20,12 @@ package minio import ( "bytes" "context" - "encoding/json" "encoding/xml" - "fmt" + "io/ioutil" "net/http" "net/url" + "strings" - "github.com/minio/minio-go/pkg/policy" "github.com/minio/minio-go/pkg/s3utils" ) @@ -100,92 +99,144 @@ func (c Client) MakeBucket(bucketName string, location string) (err error) { } // SetBucketPolicy set the access permissions on an existing bucket. -// -// For example -// -// none - owner gets full access [default]. -// readonly - anonymous get access for everyone at a given object prefix. -// readwrite - anonymous list/put/delete access to a given object prefix. -// writeonly - anonymous put/delete access to a given object prefix. -func (c Client) SetBucketPolicy(bucketName string, objectPrefix string, bucketPolicy policy.BucketPolicy) error { +func (c Client) SetBucketPolicy(bucketName, policy string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } - if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { - return err + + // If policy is empty then delete the bucket policy. + if policy == "" { + return c.removeBucketPolicy(bucketName) } - if !bucketPolicy.IsValidBucketPolicy() { - return ErrInvalidArgument(fmt.Sprintf("Invalid bucket policy provided. %s", bucketPolicy)) + // Save the updated policies. + return c.putBucketPolicy(bucketName, policy) +} + +// Saves a new bucket policy. +func (c Client) putBucketPolicy(bucketName, policy string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err } - policyInfo, err := c.getBucketPolicy(bucketName) - errResponse := ToErrorResponse(err) - if err != nil && errResponse.Code != "NoSuchBucketPolicy" { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("policy", "") + + // Content-length is mandatory for put policy request + policyReader := strings.NewReader(policy) + b, err := ioutil.ReadAll(policyReader) + if err != nil { return err } - if bucketPolicy == policy.BucketPolicyNone && policyInfo.Statements == nil { - // As the request is for removing policy and the bucket - // has empty policy statements, just return success. - return nil + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: policyReader, + contentLength: int64(len(b)), } - policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, bucketPolicy, bucketName, objectPrefix) + // Execute PUT to upload a new bucket policy. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} - // Save the updated policies. - return c.putBucketPolicy(bucketName, policyInfo) +// Removes all policies on a bucket. +func (c Client) removeBucketPolicy(bucketName string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("policy", "") + + // Execute DELETE on objectName. + resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + return nil } -// Saves a new bucket policy. -func (c Client) putBucketPolicy(bucketName string, policyInfo policy.BucketAccessPolicy) error { +// SetBucketLifecycle set the lifecycle on an existing bucket. +func (c Client) SetBucketLifecycle(bucketName, lifecycle string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } - // If there are no policy statements, we should remove entire policy. - if len(policyInfo.Statements) == 0 { - return c.removeBucketPolicy(bucketName) + // If lifecycle is empty then delete it. + if lifecycle == "" { + return c.removeBucketLifecycle(bucketName) + } + + // Save the updated lifecycle. + return c.putBucketLifecycle(bucketName, lifecycle) +} + +// Saves a new bucket lifecycle. +func (c Client) putBucketLifecycle(bucketName, lifecycle string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) - urlValues.Set("policy", "") + urlValues.Set("lifecycle", "") - policyBytes, err := json.Marshal(&policyInfo) + // Content-length is mandatory for put lifecycle request + lifecycleReader := strings.NewReader(lifecycle) + b, err := ioutil.ReadAll(lifecycleReader) if err != nil { return err } - policyBuffer := bytes.NewReader(policyBytes) reqMetadata := requestMetadata{ bucketName: bucketName, queryValues: urlValues, - contentBody: policyBuffer, - contentLength: int64(len(policyBytes)), - contentMD5Base64: sumMD5Base64(policyBytes), - contentSHA256Hex: sum256Hex(policyBytes), + contentBody: lifecycleReader, + contentLength: int64(len(b)), + contentMD5Base64: sumMD5Base64(b), } - // Execute PUT to upload a new bucket policy. + // Execute PUT to upload a new bucket lifecycle. resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp != nil { - if resp.StatusCode != http.StatusNoContent { + if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucketName, "") } } return nil } -// Removes all policies on a bucket. -func (c Client) removeBucketPolicy(bucketName string) error { +// Remove lifecycle from a bucket. +func (c Client) removeBucketLifecycle(bucketName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -193,7 +244,7 @@ func (c Client) removeBucketPolicy(bucketName string) error { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) - urlValues.Set("policy", "") + urlValues.Set("lifecycle", "") // Execute DELETE on objectName. resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ diff --git a/api-put-object-context.go b/api-put-object-context.go index a6f23dc..ff4663e 100644 --- a/api-put-object-context.go +++ b/api-put-object-context.go @@ -29,11 +29,5 @@ func (c Client) PutObjectWithContext(ctx context.Context, bucketName, objectName if err != nil { return 0, err } - if opts.EncryptMaterials != nil { - if err = opts.EncryptMaterials.SetupEncryptMode(reader); err != nil { - return 0, err - } - return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, opts.EncryptMaterials, opts) - } return c.putObjectCommon(ctx, bucketName, objectName, reader, objectSize, opts) } diff --git a/api-put-object-copy.go b/api-put-object-copy.go index 8032009..21322ef 100644 --- a/api-put-object-copy.go +++ b/api-put-object-copy.go @@ -1,6 +1,6 @@ /* * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * Copyright 2017, 2018 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,67 @@ package minio +import ( + "context" + "io" + "io/ioutil" + "net/http" + + "github.com/minio/minio-go/pkg/encrypt" +) + // CopyObject - copy a source object into a new object func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error { - return c.ComposeObject(dst, []SourceInfo{src}) + return c.CopyObjectWithProgress(dst, src, nil) +} + +// CopyObjectWithProgress - copy a source object into a new object, optionally takes +// progress bar input to notify current progress. +func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, progress io.Reader) error { + header := make(http.Header) + for k, v := range src.Headers { + header[k] = v + } + + var err error + var size int64 + // If progress bar is specified, size should be requested as well initiate a StatObject request. + if progress != nil { + size, _, _, err = src.getProps(c) + if err != nil { + return err + } + } + + if src.encryption != nil { + encrypt.SSECopy(src.encryption).Marshal(header) + } + + if dst.encryption != nil { + dst.encryption.Marshal(header) + } + for k, v := range dst.getUserMetaHeadersMap(true) { + header.Set(k, v) + } + + resp, err := c.executeMethod(context.Background(), "PUT", requestMetadata{ + bucketName: dst.bucket, + objectName: dst.object, + customHeader: header, + }) + if err != nil { + return err + } + defer closeResponse(resp) + + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, dst.bucket, dst.object) + } + + // Update the progress properly after successful copy. + if progress != nil { + io.CopyN(ioutil.Discard, progress, size) + } + + return nil } diff --git a/api-put-object-encrypted.go b/api-put-object-encrypted.go deleted file mode 100644 index 87dd1ab..0000000 --- a/api-put-object-encrypted.go +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -import ( - "context" - "io" - - "github.com/minio/minio-go/pkg/encrypt" -) - -// PutEncryptedObject - Encrypt and store object. -func (c Client) PutEncryptedObject(bucketName, objectName string, reader io.Reader, encryptMaterials encrypt.Materials) (n int64, err error) { - - if encryptMaterials == nil { - return 0, ErrInvalidArgument("Unable to recognize empty encryption properties") - } - - if err := encryptMaterials.SetupEncryptMode(reader); err != nil { - return 0, err - } - - return c.PutObjectWithContext(context.Background(), bucketName, objectName, reader, -1, PutObjectOptions{EncryptMaterials: encryptMaterials}) -} - -// FPutEncryptedObject - Encrypt and store an object with contents from file at filePath. -func (c Client) FPutEncryptedObject(bucketName, objectName, filePath string, encryptMaterials encrypt.Materials) (n int64, err error) { - return c.FPutObjectWithContext(context.Background(), bucketName, objectName, filePath, PutObjectOptions{EncryptMaterials: encryptMaterials}) -} diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index f5b8893..db92520 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -33,6 +33,7 @@ import ( "strconv" "strings" + "github.com/minio/minio-go/pkg/encrypt" "github.com/minio/minio-go/pkg/s3utils" ) @@ -138,7 +139,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje // Proceed to upload the part. var objPart ObjectPart objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, - md5Base64, sha256Hex, int64(length), opts.UserMetadata) + md5Base64, sha256Hex, int64(length), opts.ServerSideEncryption) if err != nil { return totalUploadedSize, err } @@ -226,11 +227,9 @@ func (c Client) initiateMultipartUpload(ctx context.Context, bucketName, objectN return initiateMultipartUploadResult, nil } -const serverEncryptionKeyPrefix = "x-amz-server-side-encryption" - // uploadPart - Uploads a part in a multipart upload. func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID string, reader io.Reader, - partNumber int, md5Base64, sha256Hex string, size int64, metadata map[string]string) (ObjectPart, error) { + partNumber int, md5Base64, sha256Hex string, size int64, sse encrypt.ServerSide) (ObjectPart, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ObjectPart{}, err @@ -260,12 +259,12 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID // Set encryption headers, if any. customHeader := make(http.Header) - for k, v := range metadata { - if len(v) > 0 { - if strings.HasPrefix(strings.ToLower(k), serverEncryptionKeyPrefix) { - customHeader.Set(k, v) - } - } + // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html + // Server-side encryption is supported by the S3 Multipart Upload actions. + // Unless you are using a customer-provided encryption key, you don't need + // to specify the encryption parameters in each UploadPart request. + if sse != nil && sse.Type() == encrypt.SSEC { + sse.Marshal(customHeader) } reqMetadata := requestMetadata{ diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 579cb54..211d1c2 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -167,7 +167,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa var objPart ObjectPart objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, sectionReader, uploadReq.PartNum, - "", "", partSize, opts.UserMetadata) + "", "", partSize, opts.ServerSideEncryption) if err != nil { uploadedPartsCh <- uploadedPartRes{ Size: 0, @@ -280,7 +280,7 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa var objPart ObjectPart objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, io.LimitReader(hookReader, partSize), - partNumber, "", "", partSize, opts.UserMetadata) + partNumber, "", "", partSize, opts.ServerSideEncryption) if err != nil { return totalUploadedSize, err } @@ -339,7 +339,7 @@ func (c Client) putObjectNoChecksum(ctx context.Context, bucketName, objectName // Size -1 is only supported on Google Cloud Storage, we error // out in all other situations. - if size < 0 && !s3utils.IsGoogleEndpoint(c.endpointURL) { + if size < 0 && !s3utils.IsGoogleEndpoint(*c.endpointURL) { return 0, ErrEntityTooSmall(size, bucketName, objectName) } if size > 0 { diff --git a/api-put-object.go b/api-put-object.go index 1fda1bc..0330cd9 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -28,18 +28,22 @@ import ( "github.com/minio/minio-go/pkg/encrypt" "github.com/minio/minio-go/pkg/s3utils" + "golang.org/x/net/http/httpguts" ) // PutObjectOptions represents options specified by user for PutObject call type PutObjectOptions struct { - UserMetadata map[string]string - Progress io.Reader - ContentType string - ContentEncoding string - ContentDisposition string - CacheControl string - EncryptMaterials encrypt.Materials - NumThreads uint + UserMetadata map[string]string + Progress io.Reader + ContentType string + ContentEncoding string + ContentDisposition string + ContentLanguage string + CacheControl string + ServerSideEncryption encrypt.ServerSide + NumThreads uint + StorageClass string + WebsiteRedirectLocation string } // getNumThreads - gets the number of threads to be used in the multipart @@ -69,16 +73,23 @@ func (opts PutObjectOptions) Header() (header http.Header) { if opts.ContentDisposition != "" { header["Content-Disposition"] = []string{opts.ContentDisposition} } + if opts.ContentLanguage != "" { + header["Content-Language"] = []string{opts.ContentLanguage} + } if opts.CacheControl != "" { header["Cache-Control"] = []string{opts.CacheControl} } - if opts.EncryptMaterials != nil { - header[amzHeaderIV] = []string{opts.EncryptMaterials.GetIV()} - header[amzHeaderKey] = []string{opts.EncryptMaterials.GetKey()} - header[amzHeaderMatDesc] = []string{opts.EncryptMaterials.GetDesc()} + if opts.ServerSideEncryption != nil { + opts.ServerSideEncryption.Marshal(header) + } + if opts.StorageClass != "" { + header[amzStorageClass] = []string{opts.StorageClass} + } + if opts.WebsiteRedirectLocation != "" { + header[amzWebsiteRedirectLocation] = []string{opts.WebsiteRedirectLocation} } for k, v := range opts.UserMetadata { - if !isAmzHeader(k) && !isStandardHeader(k) && !isSSEHeader(k) { + if !isAmzHeader(k) && !isStandardHeader(k) && !isStorageClassHeader(k) { header["X-Amz-Meta-"+k] = []string{v} } else { header[k] = []string{v} @@ -87,12 +98,14 @@ func (opts PutObjectOptions) Header() (header http.Header) { return } -// validate() checks if the UserMetadata map has standard headers or client side -// encryption headers and raises an error if so. +// validate() checks if the UserMetadata map has standard headers or and raises an error if so. func (opts PutObjectOptions) validate() (err error) { - for k := range opts.UserMetadata { - if isStandardHeader(k) || isCSEHeader(k) { - return ErrInvalidArgument(k + " unsupported request parameter for user defined metadata") + for k, v := range opts.UserMetadata { + if !httpguts.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) { + return ErrInvalidArgument(k + " unsupported user defined metadata name") + } + if !httpguts.ValidHeaderFieldValue(v) { + return ErrInvalidArgument(v + " unsupported user defined metadata value") } } return nil @@ -129,7 +142,7 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri } // NOTE: Streaming signature is not supported by GCS. - if s3utils.IsGoogleEndpoint(c.endpointURL) { + if s3utils.IsGoogleEndpoint(*c.endpointURL) { // Do not compute MD5 for Google Cloud Storage. return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) } @@ -199,7 +212,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName if rErr == io.EOF && partNumber > 1 { break } - if rErr != nil && rErr != io.ErrUnexpectedEOF { + if rErr != nil && rErr != io.ErrUnexpectedEOF && rErr != io.EOF { return 0, rErr } // Update progress reader appropriately to the latest offset @@ -209,7 +222,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName // Proceed to upload the part. var objPart ObjectPart objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, - "", "", int64(length), opts.UserMetadata) + "", "", int64(length), opts.ServerSideEncryption) if err != nil { return totalUploadedSize, err } diff --git a/api-put-object_test.go b/api-put-object_test.go index e0557e2..d96abab 100644 --- a/api-put-object_test.go +++ b/api-put-object_test.go @@ -22,32 +22,42 @@ import ( func TestPutObjectOptionsValidate(t *testing.T) { testCases := []struct { - metadata map[string]string - shouldPass bool + name, value string + shouldPass bool }{ - {map[string]string{"Content-Type": "custom/content-type"}, false}, - {map[string]string{"content-type": "custom/content-type"}, false}, - {map[string]string{"Content-Encoding": "gzip"}, false}, - {map[string]string{"Cache-Control": "blah"}, false}, - {map[string]string{"Content-Disposition": "something"}, false}, - {map[string]string{"my-custom-header": "blah"}, true}, - {map[string]string{"X-Amz-Iv": "blah"}, false}, - {map[string]string{"X-Amz-Key": "blah"}, false}, - {map[string]string{"X-Amz-Key-prefixed-header": "blah"}, false}, - {map[string]string{"custom-X-Amz-Key-middle": "blah"}, true}, - {map[string]string{"my-custom-header-X-Amz-Key": "blah"}, true}, - {map[string]string{"X-Amz-Matdesc": "blah"}, false}, - {map[string]string{"blah-X-Amz-Matdesc": "blah"}, true}, - {map[string]string{"X-Amz-MatDesc-suffix": "blah"}, true}, - {map[string]string{"x-amz-meta-X-Amz-Iv": "blah"}, false}, - {map[string]string{"x-amz-meta-X-Amz-Key": "blah"}, false}, - {map[string]string{"x-amz-meta-X-Amz-Matdesc": "blah"}, false}, + // Invalid cases. + {"X-Amz-Matdesc", "blah", false}, + {"x-amz-meta-X-Amz-Iv", "blah", false}, + {"x-amz-meta-X-Amz-Key", "blah", false}, + {"x-amz-meta-X-Amz-Matdesc", "blah", false}, + {"It has spaces", "v", false}, + {"It,has@illegal=characters", "v", false}, + {"X-Amz-Iv", "blah", false}, + {"X-Amz-Key", "blah", false}, + {"X-Amz-Key-prefixed-header", "blah", false}, + {"Content-Type", "custom/content-type", false}, + {"content-type", "custom/content-type", false}, + {"Content-Encoding", "gzip", false}, + {"Cache-Control", "blah", false}, + {"Content-Disposition", "something", false}, + {"Content-Language", "somelanguage", false}, + + // Valid metadata names. + {"my-custom-header", "blah", true}, + {"custom-X-Amz-Key-middle", "blah", true}, + {"my-custom-header-X-Amz-Key", "blah", true}, + {"blah-X-Amz-Matdesc", "blah", true}, + {"X-Amz-MatDesc-suffix", "blah", true}, + {"It-Is-Fine", "v", true}, + {"Numbers-098987987-Should-Work", "v", true}, + {"Crazy-!#$%&'*+-.^_`|~-Should-193832-Be-Fine", "v", true}, } for i, testCase := range testCases { - err := PutObjectOptions{UserMetadata: testCase.metadata}.validate() - + err := PutObjectOptions{UserMetadata: map[string]string{ + testCase.name: testCase.value, + }}.validate() if testCase.shouldPass && err != nil { - t.Errorf("Test %d - output did not match with reference results", i+1) + t.Errorf("Test %d - output did not match with reference results, %s", i+1, err) } } } diff --git a/api-remove.go b/api-remove.go index f14b2eb..f33df4d 100644 --- a/api-remove.go +++ b/api-remove.go @@ -129,10 +129,8 @@ func processRemoveMultiObjectsResponse(body io.Reader, objects []string, errorCh } } -// RemoveObjects remove multiples objects from a bucket. -// The list of objects to remove are received from objectsCh. -// Remove failures are sent back via error channel. -func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan RemoveObjectError { +// RemoveObjectsWithContext - Identical to RemoveObjects call, but accepts context to facilitate request cancellation. +func (c Client) RemoveObjectsWithContext(ctx context.Context, bucketName string, objectsCh <-chan string) <-chan RemoveObjectError { errorCh := make(chan RemoveObjectError, 1) // Validate if bucket name is valid. @@ -189,7 +187,7 @@ func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan // Generate remove multi objects XML request removeBytes := generateRemoveMultiObjectsRequest(batch) // Execute GET on bucket to list objects. - resp, err := c.executeMethod(context.Background(), "POST", requestMetadata{ + resp, err := c.executeMethod(ctx, "POST", requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentBody: bytes.NewReader(removeBytes), @@ -197,6 +195,12 @@ func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan contentMD5Base64: sumMD5Base64(removeBytes), contentSHA256Hex: sum256Hex(removeBytes), }) + if resp != nil { + if resp.StatusCode != http.StatusOK { + e := httpRespToErrorResponse(resp, bucketName, "") + errorCh <- RemoveObjectError{ObjectName: "", Err: e} + } + } if err != nil { for _, b := range batch { errorCh <- RemoveObjectError{ObjectName: b, Err: err} @@ -213,6 +217,13 @@ func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan return errorCh } +// RemoveObjects removes multiple objects from a bucket. +// The list of objects to remove are received from objectsCh. +// Remove failures are sent back via error channel. +func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan RemoveObjectError { + return c.RemoveObjectsWithContext(context.Background(), bucketName, objectsCh) +} + // RemoveIncompleteUpload aborts an partially uploaded object. func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { // Input validation. @@ -222,18 +233,20 @@ func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { if err := s3utils.CheckValidObjectName(objectName); err != nil { return err } - // Find multipart upload id of the object to be aborted. - uploadID, err := c.findUploadID(bucketName, objectName) + // Find multipart upload ids of the object to be aborted. + uploadIDs, err := c.findUploadIDs(bucketName, objectName) if err != nil { return err } - if uploadID != "" { - // Upload id found, abort the incomplete multipart upload. + + for _, uploadID := range uploadIDs { + // abort incomplete multipart upload, based on the upload id passed. err := c.abortMultipartUpload(context.Background(), bucketName, objectName, uploadID) if err != nil { return err } } + return nil } diff --git a/api-select.go b/api-select.go new file mode 100644 index 0000000..a9b6f17 --- /dev/null +++ b/api-select.go @@ -0,0 +1,520 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/xml" + "errors" + "fmt" + "hash" + "hash/crc32" + "io" + "net/http" + "net/url" + "strings" + + "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/pkg/s3utils" +) + +// CSVFileHeaderInfo - is the parameter for whether to utilize headers. +type CSVFileHeaderInfo string + +// Constants for file header info. +const ( + CSVFileHeaderInfoNone CSVFileHeaderInfo = "NONE" + CSVFileHeaderInfoIgnore = "IGNORE" + CSVFileHeaderInfoUse = "USE" +) + +// SelectCompressionType - is the parameter for what type of compression is +// present +type SelectCompressionType string + +// Constants for compression types under select API. +const ( + SelectCompressionNONE SelectCompressionType = "NONE" + SelectCompressionGZIP = "GZIP" + SelectCompressionBZIP = "BZIP2" +) + +// CSVQuoteFields - is the parameter for how CSV fields are quoted. +type CSVQuoteFields string + +// Constants for csv quote styles. +const ( + CSVQuoteFieldsAlways CSVQuoteFields = "Always" + CSVQuoteFieldsAsNeeded = "AsNeeded" +) + +// QueryExpressionType - is of what syntax the expression is, this should only +// be SQL +type QueryExpressionType string + +// Constants for expression type. +const ( + QueryExpressionTypeSQL QueryExpressionType = "SQL" +) + +// JSONType determines json input serialization type. +type JSONType string + +// Constants for JSONTypes. +const ( + JSONDocumentType JSONType = "DOCUMENT" + JSONLinesType = "LINES" +) + +// ParquetInputOptions parquet input specific options +type ParquetInputOptions struct{} + +// CSVInputOptions csv input specific options +type CSVInputOptions struct { + FileHeaderInfo CSVFileHeaderInfo + RecordDelimiter string + FieldDelimiter string + QuoteCharacter string + QuoteEscapeCharacter string + Comments string +} + +// CSVOutputOptions csv output specific options +type CSVOutputOptions struct { + QuoteFields CSVQuoteFields + RecordDelimiter string + FieldDelimiter string + QuoteCharacter string + QuoteEscapeCharacter string +} + +// JSONInputOptions json input specific options +type JSONInputOptions struct { + Type JSONType +} + +// JSONOutputOptions - json output specific options +type JSONOutputOptions struct { + RecordDelimiter string +} + +// SelectObjectInputSerialization - input serialization parameters +type SelectObjectInputSerialization struct { + CompressionType SelectCompressionType + Parquet *ParquetInputOptions `xml:"Parquet,omitempty"` + CSV *CSVInputOptions `xml:"CSV,omitempty"` + JSON *JSONInputOptions `xml:"JSON,omitempty"` +} + +// SelectObjectOutputSerialization - output serialization parameters. +type SelectObjectOutputSerialization struct { + CSV *CSVOutputOptions `xml:"CSV,omitempty"` + JSON *JSONOutputOptions `xml:"JSON,omitempty"` +} + +// SelectObjectOptions - represents the input select body +type SelectObjectOptions struct { + XMLName xml.Name `xml:"SelectObjectContentRequest" json:"-"` + ServerSideEncryption encrypt.ServerSide `xml:"-"` + Expression string + ExpressionType QueryExpressionType + InputSerialization SelectObjectInputSerialization + OutputSerialization SelectObjectOutputSerialization + RequestProgress struct { + Enabled bool + } +} + +// Header returns the http.Header representation of the SelectObject options. +func (o SelectObjectOptions) Header() http.Header { + headers := make(http.Header) + if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() == encrypt.SSEC { + o.ServerSideEncryption.Marshal(headers) + } + return headers +} + +// SelectObjectType - is the parameter which defines what type of object the +// operation is being performed on. +type SelectObjectType string + +// Constants for input data types. +const ( + SelectObjectTypeCSV SelectObjectType = "CSV" + SelectObjectTypeJSON = "JSON" + SelectObjectTypeParquet = "Parquet" +) + +// preludeInfo is used for keeping track of necessary information from the +// prelude. +type preludeInfo struct { + totalLen uint32 + headerLen uint32 +} + +// SelectResults is used for the streaming responses from the server. +type SelectResults struct { + pipeReader *io.PipeReader + resp *http.Response + stats *StatsMessage + progress *ProgressMessage +} + +// ProgressMessage is a struct for progress xml message. +type ProgressMessage struct { + XMLName xml.Name `xml:"Progress" json:"-"` + StatsMessage +} + +// StatsMessage is a struct for stat xml message. +type StatsMessage struct { + XMLName xml.Name `xml:"Stats" json:"-"` + BytesScanned int64 + BytesProcessed int64 + BytesReturned int64 +} + +// eventType represents the type of event. +type eventType string + +// list of event-types returned by Select API. +const ( + endEvent eventType = "End" + errorEvent = "Error" + recordsEvent = "Records" + progressEvent = "Progress" + statsEvent = "Stats" +) + +// contentType represents content type of event. +type contentType string + +const ( + xmlContent contentType = "text/xml" +) + +// SelectObjectContent is a implementation of http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html AWS S3 API. +func (c Client) SelectObjectContent(ctx context.Context, bucketName, objectName string, opts SelectObjectOptions) (*SelectResults, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return nil, err + } + + selectReqBytes, err := xml.Marshal(opts) + if err != nil { + return nil, err + } + + urlValues := make(url.Values) + urlValues.Set("select", "") + urlValues.Set("select-type", "2") + + // Execute POST on bucket/object. + resp, err := c.executeMethod(ctx, "POST", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + customHeader: opts.Header(), + contentMD5Base64: sumMD5Base64(selectReqBytes), + contentSHA256Hex: sum256Hex(selectReqBytes), + contentBody: bytes.NewReader(selectReqBytes), + contentLength: int64(len(selectReqBytes)), + }) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, "") + } + + pipeReader, pipeWriter := io.Pipe() + streamer := &SelectResults{ + resp: resp, + stats: &StatsMessage{}, + progress: &ProgressMessage{}, + pipeReader: pipeReader, + } + streamer.start(pipeWriter) + return streamer, nil +} + +// Close - closes the underlying response body and the stream reader. +func (s *SelectResults) Close() error { + defer closeResponse(s.resp) + return s.pipeReader.Close() +} + +// Read - is a reader compatible implementation for SelectObjectContent records. +func (s *SelectResults) Read(b []byte) (n int, err error) { + return s.pipeReader.Read(b) +} + +// Stats - information about a request's stats when processing is complete. +func (s *SelectResults) Stats() *StatsMessage { + return s.stats +} + +// Progress - information about the progress of a request. +func (s *SelectResults) Progress() *ProgressMessage { + return s.progress +} + +// start is the main function that decodes the large byte array into +// several events that are sent through the eventstream. +func (s *SelectResults) start(pipeWriter *io.PipeWriter) { + go func() { + for { + var prelude preludeInfo + var headers = make(http.Header) + var err error + + // Create CRC code + crc := crc32.New(crc32.IEEETable) + crcReader := io.TeeReader(s.resp.Body, crc) + + // Extract the prelude(12 bytes) into a struct to extract relevant information. + prelude, err = processPrelude(crcReader, crc) + if err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + + // Extract the headers(variable bytes) into a struct to extract relevant information + if prelude.headerLen > 0 { + if err = extractHeader(io.LimitReader(crcReader, int64(prelude.headerLen)), headers); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + } + + // Get the actual payload length so that the appropriate amount of + // bytes can be read or parsed. + payloadLen := prelude.PayloadLen() + + // Get content-type of the payload. + c := contentType(headers.Get("content-type")) + + // Get event type of the payload. + e := eventType(headers.Get("event-type")) + + // Handle all supported events. + switch e { + case endEvent: + pipeWriter.Close() + closeResponse(s.resp) + return + case errorEvent: + pipeWriter.CloseWithError(errors.New("Error Type of " + headers.Get("error-type") + " " + headers.Get("error-message"))) + closeResponse(s.resp) + return + case recordsEvent: + if _, err = io.Copy(pipeWriter, io.LimitReader(crcReader, payloadLen)); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + case progressEvent: + switch c { + case xmlContent: + if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.progress); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + default: + pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, progressEvent)) + closeResponse(s.resp) + return + } + case statsEvent: + switch c { + case xmlContent: + if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.stats); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + default: + pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, statsEvent)) + closeResponse(s.resp) + return + } + } + + // Ensures that the full message's CRC is correct and + // that the message is not corrupted + if err := checkCRC(s.resp.Body, crc.Sum32()); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + + } + }() +} + +// PayloadLen is a function that calculates the length of the payload. +func (p preludeInfo) PayloadLen() int64 { + return int64(p.totalLen - p.headerLen - 16) +} + +// processPrelude is the function that reads the 12 bytes of the prelude and +// ensures the CRC is correct while also extracting relevant information into +// the struct, +func processPrelude(prelude io.Reader, crc hash.Hash32) (preludeInfo, error) { + var err error + var pInfo = preludeInfo{} + + // reads total length of the message (first 4 bytes) + pInfo.totalLen, err = extractUint32(prelude) + if err != nil { + return pInfo, err + } + + // reads total header length of the message (2nd 4 bytes) + pInfo.headerLen, err = extractUint32(prelude) + if err != nil { + return pInfo, err + } + + // checks that the CRC is correct (3rd 4 bytes) + preCRC := crc.Sum32() + if err := checkCRC(prelude, preCRC); err != nil { + return pInfo, err + } + + return pInfo, nil +} + +// extracts the relevant information from the Headers. +func extractHeader(body io.Reader, myHeaders http.Header) error { + for { + // extracts the first part of the header, + headerTypeName, err := extractHeaderType(body) + if err != nil { + // Since end of file, we have read all of our headers + if err == io.EOF { + break + } + return err + } + + // reads the 7 present in the header and ignores it. + extractUint8(body) + + headerValueName, err := extractHeaderValue(body) + if err != nil { + return err + } + + myHeaders.Set(headerTypeName, headerValueName) + + } + return nil +} + +// extractHeaderType extracts the first half of the header message, the header type. +func extractHeaderType(body io.Reader) (string, error) { + // extracts 2 bit integer + headerNameLen, err := extractUint8(body) + if err != nil { + return "", err + } + // extracts the string with the appropriate number of bytes + headerName, err := extractString(body, int(headerNameLen)) + if err != nil { + return "", err + } + return strings.TrimPrefix(headerName, ":"), nil +} + +// extractsHeaderValue extracts the second half of the header message, the +// header value +func extractHeaderValue(body io.Reader) (string, error) { + bodyLen, err := extractUint16(body) + if err != nil { + return "", err + } + bodyName, err := extractString(body, int(bodyLen)) + if err != nil { + return "", err + } + return bodyName, nil +} + +// extracts a string from byte array of a particular number of bytes. +func extractString(source io.Reader, lenBytes int) (string, error) { + myVal := make([]byte, lenBytes) + _, err := source.Read(myVal) + if err != nil { + return "", err + } + return string(myVal), nil +} + +// extractUint32 extracts a 4 byte integer from the byte array. +func extractUint32(r io.Reader) (uint32, error) { + buf := make([]byte, 4) + _, err := io.ReadFull(r, buf) + if err != nil { + return 0, err + } + return binary.BigEndian.Uint32(buf), nil +} + +// extractUint16 extracts a 2 byte integer from the byte array. +func extractUint16(r io.Reader) (uint16, error) { + buf := make([]byte, 2) + _, err := io.ReadFull(r, buf) + if err != nil { + return 0, err + } + return binary.BigEndian.Uint16(buf), nil +} + +// extractUint8 extracts a 1 byte integer from the byte array. +func extractUint8(r io.Reader) (uint8, error) { + buf := make([]byte, 1) + _, err := io.ReadFull(r, buf) + if err != nil { + return 0, err + } + return buf[0], nil +} + +// checkCRC ensures that the CRC matches with the one from the reader. +func checkCRC(r io.Reader, expect uint32) error { + msgCRC, err := extractUint32(r) + if err != nil { + return err + } + + if msgCRC != expect { + return fmt.Errorf("Checksum Mismatch, MessageCRC of 0x%X does not equal expected CRC of 0x%X", msgCRC, expect) + + } + return nil +} diff --git a/api-stat.go b/api-stat.go index 8904dd6..91e9d39 100644 --- a/api-stat.go +++ b/api-stat.go @@ -47,6 +47,10 @@ func (c Client) BucketExists(bucketName string) (bool, error) { return false, err } if resp != nil { + resperr := httpRespToErrorResponse(resp, bucketName, "") + if ToErrorResponse(resperr).Code == "NoSuchBucket" { + return false, nil + } if resp.StatusCode != http.StatusOK { return false, httpRespToErrorResponse(resp, bucketName, "") } @@ -66,6 +70,9 @@ var defaultFilterKeys = []string{ "x-amz-bucket-region", "x-amz-request-id", "x-amz-id-2", + "Content-Security-Policy", + "X-Xss-Protection", + // Add new headers to be ignored. } @@ -115,7 +122,7 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o return ObjectInfo{}, err } if resp != nil { - if resp.StatusCode != http.StatusOK { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) } } diff --git a/api.go b/api.go index 9951d47..79b0bc2 100644 --- a/api.go +++ b/api.go @@ -1,6 +1,6 @@ /* * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * Copyright 2015-2018 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,7 +48,7 @@ type Client struct { /// Standard options. // Parsed endpoint url provided by the user. - endpointURL url.URL + endpointURL *url.URL // Holds various credential providers. credsProvider *credentials.Credentials @@ -81,12 +81,25 @@ type Client struct { // Random seed. random *rand.Rand + + // lookup indicates type of url lookup supported by server. If not specified, + // default to Auto. + lookup BucketLookupType +} + +// Options for New method +type Options struct { + Creds *credentials.Credentials + Secure bool + Region string + BucketLookup BucketLookupType + // Add future fields here } // Global constants. const ( libraryName = "minio-go" - libraryVersion = "4.0.5" + libraryVersion = "v6.0.11" ) // User Agent should always following the below style. @@ -98,11 +111,21 @@ const ( libraryUserAgent = libraryUserAgentPrefix + libraryName + "/" + libraryVersion ) +// BucketLookupType is type of url lookup supported by server. +type BucketLookupType int + +// Different types of url lookup supported by the server.Initialized to BucketLookupAuto +const ( + BucketLookupAuto BucketLookupType = iota + BucketLookupDNS + BucketLookupPath +) + // NewV2 - instantiate minio client with Amazon S3 signature version // '2' compatibility. func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { creds := credentials.NewStaticV2(accessKeyID, secretAccessKey, "") - clnt, err := privateNew(endpoint, creds, secure, "") + clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto) if err != nil { return nil, err } @@ -114,7 +137,7 @@ func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (* // '4' compatibility. func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") - clnt, err := privateNew(endpoint, creds, secure, "") + clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto) if err != nil { return nil, err } @@ -125,16 +148,16 @@ func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (* // New - instantiate minio client, adds automatic verification of signature. func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") - clnt, err := privateNew(endpoint, creds, secure, "") + clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto) if err != nil { return nil, err } // Google cloud storage should be set to signature V2, force it if not. - if s3utils.IsGoogleEndpoint(clnt.endpointURL) { + if s3utils.IsGoogleEndpoint(*clnt.endpointURL) { clnt.overrideSignerType = credentials.SignatureV2 } // If Amazon S3 set to signature v4. - if s3utils.IsAmazonEndpoint(clnt.endpointURL) { + if s3utils.IsAmazonEndpoint(*clnt.endpointURL) { clnt.overrideSignerType = credentials.SignatureV4 } return clnt, nil @@ -144,7 +167,7 @@ func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, e // for retrieving credentials from various credentials provider such as // IAM, File, Env etc. func NewWithCredentials(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) { - return privateNew(endpoint, creds, secure, region) + return privateNew(endpoint, creds, secure, region, BucketLookupAuto) } // NewWithRegion - instantiate minio client, with region configured. Unlike New(), @@ -152,7 +175,12 @@ func NewWithCredentials(endpoint string, creds *credentials.Credentials, secure // Use this function when if your application deals with single region. func NewWithRegion(endpoint, accessKeyID, secretAccessKey string, secure bool, region string) (*Client, error) { creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") - return privateNew(endpoint, creds, secure, region) + return privateNew(endpoint, creds, secure, region, BucketLookupAuto) +} + +// NewWithOptions - instantiate minio client with options +func NewWithOptions(endpoint string, opts *Options) (*Client, error) { + return privateNew(endpoint, opts.Creds, opts.Secure, opts.Region, opts.BucketLookup) } // lockedRandSource provides protected rand source, implements rand.Source interface. @@ -177,32 +205,68 @@ func (r *lockedRandSource) Seed(seed int64) { r.lk.Unlock() } -// getRegionFromURL - parse region from URL if present. -func getRegionFromURL(u url.URL) (region string) { - region = "" - if s3utils.IsGoogleEndpoint(u) { - return - } else if s3utils.IsAmazonChinaEndpoint(u) { - // For china specifically we need to set everything to - // cn-north-1 for now, there is no easier way until AWS S3 - // provides a cleaner compatible API across "us-east-1" and - // China region. - return "cn-north-1" - } else if s3utils.IsAmazonGovCloudEndpoint(u) { - // For us-gov specifically we need to set everything to - // us-gov-west-1 for now, there is no easier way until AWS S3 - // provides a cleaner compatible API across "us-east-1" and - // Gov cloud region. - return "us-gov-west-1" - } - parts := s3utils.AmazonS3Host.FindStringSubmatch(u.Host) - if len(parts) > 1 { - region = parts[1] - } - return region +// Redirect requests by re signing the request. +func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error { + if len(via) >= 5 { + return errors.New("stopped after 5 redirects") + } + if len(via) == 0 { + return nil + } + lastRequest := via[len(via)-1] + var reAuth bool + for attr, val := range lastRequest.Header { + // if hosts do not match do not copy Authorization header + if attr == "Authorization" && req.Host != lastRequest.Host { + reAuth = true + continue + } + if _, ok := req.Header[attr]; !ok { + req.Header[attr] = val + } + } + + *c.endpointURL = *req.URL + + value, err := c.credsProvider.Get() + if err != nil { + return err + } + var ( + signerType = value.SignerType + accessKeyID = value.AccessKeyID + secretAccessKey = value.SecretAccessKey + sessionToken = value.SessionToken + region = c.region + ) + + // Custom signer set then override the behavior. + if c.overrideSignerType != credentials.SignatureDefault { + signerType = c.overrideSignerType + } + + // If signerType returned by credentials helper is anonymous, + // then do not sign regardless of signerType override. + if value.SignerType == credentials.SignatureAnonymous { + signerType = credentials.SignatureAnonymous + } + + if reAuth { + // Check if there is no region override, if not get it from the URL if possible. + if region == "" { + region = s3utils.GetRegionFromURL(*c.endpointURL) + } + switch { + case signerType.IsV2(): + return errors.New("signature V2 cannot support redirection") + case signerType.IsV4(): + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region)) + } + } + return nil } -func privateNew(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) { +func privateNew(endpoint string, creds *credentials.Credentials, secure bool, region string, lookup BucketLookupType) (*Client, error) { // construct endpoint. endpointURL, err := getEndpointURL(endpoint, secure) if err != nil { @@ -219,16 +283,17 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re clnt.secure = secure // Save endpoint URL, user agent for future uses. - clnt.endpointURL = *endpointURL + clnt.endpointURL = endpointURL // Instantiate http client and bucket location cache. clnt.httpClient = &http.Client{ - Transport: defaultMinioTransport, + Transport: DefaultTransport, + CheckRedirect: clnt.redirectHeaders, } // Sets custom region, if region is empty bucket location cache is used automatically. if region == "" { - region = getRegionFromURL(clnt.endpointURL) + region = s3utils.GetRegionFromURL(*clnt.endpointURL) } clnt.region = region @@ -238,6 +303,9 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re // Introduce a new locked random seed. clnt.random = rand.New(&lockedRandSource{src: rand.NewSource(time.Now().UTC().UnixNano())}) + // Sets bucket lookup style, whether server accepts DNS or Path lookup. Default is Auto - determined + // by the SDK. When Auto is specified, DNS lookup is used for Amazon/Google cloud endpoints and Path for all other endpoints. + clnt.lookup = lookup // Return. return clnt, nil } @@ -269,7 +337,7 @@ func (c *Client) SetCustomTransport(customHTTPTransport http.RoundTripper) { // TLSClientConfig: &tls.Config{RootCAs: pool}, // DisableCompression: true, // } - // api.SetTransport(tr) + // api.SetCustomTransport(tr) // if c.httpClient != nil { c.httpClient.Transport = customHTTPTransport @@ -301,7 +369,7 @@ func (c *Client) TraceOff() { // please vist - // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) { - if s3utils.IsAmazonEndpoint(c.endpointURL) { + if s3utils.IsAmazonEndpoint(*c.endpointURL) { c.s3AccelerateEndpoint = accelerateEndpoint } } @@ -387,24 +455,12 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { return err } } else { - // WORKAROUND for https://github.com/golang/go/issues/13942. - // httputil.DumpResponse does not print response headers for - // all successful calls which have response ContentLength set - // to zero. Keep this workaround until the above bug is fixed. - if resp.ContentLength == 0 { - var buffer bytes.Buffer - if err = resp.Header.Write(&buffer); err != nil { - return err - } - respTrace = buffer.Bytes() - respTrace = append(respTrace, []byte("\r\n")...) - } else { - respTrace, err = httputil.DumpResponse(resp, false) - if err != nil { - return err - } + respTrace, err = httputil.DumpResponse(resp, false) + if err != nil { + return err } } + // Write response to trace output. _, err = fmt.Fprint(c.traceOutput, strings.TrimSuffix(string(respTrace), "\r\n")) if err != nil { @@ -423,38 +479,22 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { // do - execute http request. func (c Client) do(req *http.Request) (*http.Response, error) { - var resp *http.Response - var err error - // Do the request in a loop in case of 307 http is met since golang still doesn't - // handle properly this situation (https://github.com/golang/go/issues/7912) - for { - resp, err = c.httpClient.Do(req) - if err != nil { - // Handle this specifically for now until future Golang - // versions fix this issue properly. - urlErr, ok := err.(*url.Error) - if ok && strings.Contains(urlErr.Err.Error(), "EOF") { + resp, err := c.httpClient.Do(req) + if err != nil { + // Handle this specifically for now until future Golang versions fix this issue properly. + if urlErr, ok := err.(*url.Error); ok { + if strings.Contains(urlErr.Err.Error(), "EOF") { return nil, &url.Error{ Op: urlErr.Op, URL: urlErr.URL, Err: errors.New("Connection closed by foreign host " + urlErr.URL + ". Retry again."), } } - return nil, err - } - // Redo the request with the new redirect url if http 307 is returned, quit the loop otherwise - if resp != nil && resp.StatusCode == http.StatusTemporaryRedirect { - newURL, err := url.Parse(resp.Header.Get("Location")) - if err != nil { - break - } - req.URL = newURL - } else { - break } + return nil, err } - // Response cannot be non-nil, report if its the case. + // Response cannot be non-nil, report error if thats the case. if resp == nil { msg := "Response is empty. " + reportIssue return nil, ErrInvalidArgument(msg) @@ -467,6 +507,7 @@ func (c Client) do(req *http.Request) (*http.Response, error) { return nil, err } } + return resp, nil } @@ -538,14 +579,15 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque } return nil, err } + // Add context to request req = req.WithContext(ctx) // Initiate the request. res, err = c.do(req) if err != nil { - // For supported network errors verify. - if isNetErrorRetryable(err) { + // For supported http requests errors verify. + if isHTTPReqErrorRetryable(err) { continue // Retry. } // For other errors, return here no need to retry. @@ -634,12 +676,15 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R // happen when GetBucketLocation() is disabled using IAM policies. } if location == "" { - location = getDefaultLocation(c.endpointURL, c.region) + location = getDefaultLocation(*c.endpointURL, c.region) } } + // Look if target url supports virtual host. + isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, metadata.bucketName) + // Construct a new target URL. - targetURL, err := c.makeTargetURL(metadata.bucketName, metadata.objectName, location, metadata.queryValues) + targetURL, err := c.makeTargetURL(metadata.bucketName, metadata.objectName, location, isVirtualHost, metadata.queryValues) if err != nil { return nil, err } @@ -681,7 +726,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R } if signerType.IsV2() { // Presign URL with signature v2. - req = s3signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires) + req = s3signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost) } else if signerType.IsV4() { // Presign URL with signature v4. req = s3signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires) @@ -727,7 +772,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R switch { case signerType.IsV2(): // Add signature version '2' authorization header. - req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) case metadata.objectName != "" && method == "PUT" && metadata.customHeader.Get("X-Amz-Copy-Source") == "" && !c.secure: // Streaming signature is used by default for a PUT object request. Additionally we also // look if the initialized client is secure, if yes then we don't need to perform @@ -759,10 +804,10 @@ func (c Client) setUserAgent(req *http.Request) { } // makeTargetURL make a new target url. -func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, queryValues url.Values) (*url.URL, error) { +func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error) { host := c.endpointURL.Host // For Amazon S3 endpoint, try to fetch location based endpoint. - if s3utils.IsAmazonEndpoint(c.endpointURL) { + if s3utils.IsAmazonEndpoint(*c.endpointURL) { if c.s3AccelerateEndpoint != "" && bucketName != "" { // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html // Disable transfer acceleration for non-compliant bucket names. @@ -775,7 +820,7 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que host = c.s3AccelerateEndpoint } else { // Do not change the host if the endpoint URL is a FIPS S3 endpoint. - if !s3utils.IsAmazonFIPSGovCloudEndpoint(c.endpointURL) { + if !s3utils.IsAmazonFIPSEndpoint(*c.endpointURL) { // Fetch new host based on the bucket location. host = getS3Endpoint(bucketLocation) } @@ -798,9 +843,6 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que // Make URL only if bucketName is available, otherwise use the // endpoint URL. if bucketName != "" { - // Save if target url will have buckets which suppport virtual host. - isVirtualHostStyle := s3utils.IsVirtualHostSupported(c.endpointURL, bucketName) - // If endpoint supports virtual host style use that always. // Currently only S3 and Google Cloud Storage would support // virtual host style. @@ -823,10 +865,23 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que urlStr = urlStr + "?" + s3utils.QueryEncode(queryValues) } - u, err := url.Parse(urlStr) - if err != nil { - return nil, err + return url.Parse(urlStr) +} + +// returns true if virtual hosted style requests are to be used. +func (c *Client) isVirtualHostStyleRequest(url url.URL, bucketName string) bool { + if bucketName == "" { + return false + } + + if c.lookup == BucketLookupDNS { + return true + } + if c.lookup == BucketLookupPath { + return false } - return u, nil + // default to virtual only for Amazon/Google storage. In all other cases use + // path style requests + return s3utils.IsVirtualHostSupported(url, bucketName) } diff --git a/api_unit_test.go b/api_unit_test.go index 96fd8dd..d7822ab 100644 --- a/api_unit_test.go +++ b/api_unit_test.go @@ -36,50 +36,6 @@ func (c *customReader) Size() (n int64) { return 10 } -// Tests get region from host URL. -func TestGetRegionFromURL(t *testing.T) { - testCases := []struct { - u url.URL - expectedRegion string - }{ - { - u: url.URL{Host: "storage.googleapis.com"}, - expectedRegion: "", - }, - { - u: url.URL{Host: "s3.cn-north-1.amazonaws.com.cn"}, - expectedRegion: "cn-north-1", - }, - { - u: url.URL{Host: "s3-fips-us-gov-west-1.amazonaws.com"}, - expectedRegion: "us-gov-west-1", - }, - { - u: url.URL{Host: "s3-us-gov-west-1.amazonaws.com"}, - expectedRegion: "us-gov-west-1", - }, - { - u: url.URL{Host: "192.168.1.1"}, - expectedRegion: "", - }, - { - u: url.URL{Host: "s3-eu-west-1.amazonaws.com"}, - expectedRegion: "eu-west-1", - }, - { - u: url.URL{Host: "s3.amazonaws.com"}, - expectedRegion: "", - }, - } - - for i, testCase := range testCases { - region := getRegionFromURL(testCase.u) - if testCase.expectedRegion != region { - t.Errorf("Test %d: Expected region %s, got %s", i+1, testCase.expectedRegion, region) - } - } -} - // Tests valid hosts for location. func TestValidBucketLocation(t *testing.T) { s3Hosts := []struct { @@ -234,7 +190,8 @@ func TestMakeTargetURL(t *testing.T) { for i, testCase := range testCases { // Initialize a Minio client c, _ := New(testCase.addr, "foo", "bar", testCase.secure) - u, err := c.makeTargetURL(testCase.bucketName, testCase.objectName, testCase.bucketLocation, testCase.queryValues) + isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, testCase.bucketName) + u, err := c.makeTargetURL(testCase.bucketName, testCase.objectName, testCase.bucketLocation, isVirtualHost, testCase.queryValues) // Check the returned error if testCase.expectedErr == nil && err != nil { t.Fatalf("Test %d: Should succeed but failed with err = %v", i+1, err) diff --git a/appveyor.yml b/appveyor.yml index 79c7a15..48ea6e7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,12 +16,11 @@ install: - set PATH=%GOPATH%\bin;c:\go\bin;%PATH% - go version - go env - - go get -u github.com/golang/lint/golint - - go get -u github.com/go-ini/ini - - go get -u github.com/mitchellh/go-homedir + - go get -u golang.org/x/lint/golint - go get -u github.com/remyoudompheng/go-misc/deadcode - go get -u github.com/gordonklaus/ineffassign - - go get -u github.com/dustin/go-humanize + - go get -u golang.org/x/crypto/argon2 + - go get -t ./... # to run your custom scripts instead of automatic MSBuild build_script: diff --git a/bucket-cache.go b/bucket-cache.go index 5d56cdf..cac7ad7 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -203,7 +203,9 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro } if signerType.IsV2() { - req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) + // Get Bucket Location calls should be always path style + isVirtualHost := false + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) return req, nil } diff --git a/bucket-cache_test.go b/bucket-cache_test.go index fd7e7f3..507b408 100644 --- a/bucket-cache_test.go +++ b/bucket-cache_test.go @@ -122,7 +122,7 @@ func TestGetBucketLocationRequest(t *testing.T) { req.Header.Set("X-Amz-Content-Sha256", contentSha256) req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") case signerType.IsV2(): - req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, false) } return req, nil diff --git a/bucket-notification.go b/bucket-notification.go index 1b9d6a0..ea303dd 100644 --- a/bucket-notification.go +++ b/bucket-notification.go @@ -19,7 +19,8 @@ package minio import ( "encoding/xml" - "reflect" + + "github.com/minio/minio-go/pkg/set" ) // NotificationEventType is a S3 notification event associated to the bucket notification configuration @@ -96,7 +97,7 @@ type NotificationConfig struct { // NewNotificationConfig creates one notification config and sets the given ARN func NewNotificationConfig(arn Arn) NotificationConfig { - return NotificationConfig{Arn: arn} + return NotificationConfig{Arn: arn, Filter: &Filter{}} } // AddEvents adds one event to the current notification config @@ -163,39 +164,79 @@ type BucketNotification struct { } // AddTopic adds a given topic config to the general bucket notification config -func (b *BucketNotification) AddTopic(topicConfig NotificationConfig) { +func (b *BucketNotification) AddTopic(topicConfig NotificationConfig) bool { newTopicConfig := TopicConfig{NotificationConfig: topicConfig, Topic: topicConfig.Arn.String()} for _, n := range b.TopicConfigs { - if reflect.DeepEqual(n, newTopicConfig) { - // Avoid adding duplicated entry - return + // If new config matches existing one + if n.Topic == newTopicConfig.Arn.String() && newTopicConfig.Filter == n.Filter { + + existingConfig := set.NewStringSet() + for _, v := range n.Events { + existingConfig.Add(string(v)) + } + + newConfig := set.NewStringSet() + for _, v := range topicConfig.Events { + newConfig.Add(string(v)) + } + + if !newConfig.Intersection(existingConfig).IsEmpty() { + return false + } } } b.TopicConfigs = append(b.TopicConfigs, newTopicConfig) + return true } // AddQueue adds a given queue config to the general bucket notification config -func (b *BucketNotification) AddQueue(queueConfig NotificationConfig) { +func (b *BucketNotification) AddQueue(queueConfig NotificationConfig) bool { newQueueConfig := QueueConfig{NotificationConfig: queueConfig, Queue: queueConfig.Arn.String()} for _, n := range b.QueueConfigs { - if reflect.DeepEqual(n, newQueueConfig) { - // Avoid adding duplicated entry - return + if n.Queue == newQueueConfig.Arn.String() && newQueueConfig.Filter == n.Filter { + + existingConfig := set.NewStringSet() + for _, v := range n.Events { + existingConfig.Add(string(v)) + } + + newConfig := set.NewStringSet() + for _, v := range queueConfig.Events { + newConfig.Add(string(v)) + } + + if !newConfig.Intersection(existingConfig).IsEmpty() { + return false + } } } b.QueueConfigs = append(b.QueueConfigs, newQueueConfig) + return true } // AddLambda adds a given lambda config to the general bucket notification config -func (b *BucketNotification) AddLambda(lambdaConfig NotificationConfig) { +func (b *BucketNotification) AddLambda(lambdaConfig NotificationConfig) bool { newLambdaConfig := LambdaConfig{NotificationConfig: lambdaConfig, Lambda: lambdaConfig.Arn.String()} for _, n := range b.LambdaConfigs { - if reflect.DeepEqual(n, newLambdaConfig) { - // Avoid adding duplicated entry - return + if n.Lambda == newLambdaConfig.Arn.String() && newLambdaConfig.Filter == n.Filter { + + existingConfig := set.NewStringSet() + for _, v := range n.Events { + existingConfig.Add(string(v)) + } + + newConfig := set.NewStringSet() + for _, v := range lambdaConfig.Events { + newConfig.Add(string(v)) + } + + if !newConfig.Intersection(existingConfig).IsEmpty() { + return false + } } } b.LambdaConfigs = append(b.LambdaConfigs, newLambdaConfig) + return true } // RemoveTopicByArn removes all topic configurations that match the exact specified ARN diff --git a/constants.go b/constants.go index b5945e7..7377423 100644 --- a/constants.go +++ b/constants.go @@ -27,10 +27,6 @@ const absMinPartSize = 1024 * 1024 * 5 // putObject behaves internally as multipart. const minPartSize = 1024 * 1024 * 64 -// copyPartSize - default (and maximum) part size to copy in a -// copy-object request (5GiB) -const copyPartSize = 1024 * 1024 * 1024 * 5 - // maxPartsCount - maximum number of parts for a single multipart session. const maxPartsCount = 10000 @@ -59,9 +55,8 @@ const ( iso8601DateFormat = "20060102T150405Z" ) -// Encryption headers stored along with the object. -const ( - amzHeaderIV = "X-Amz-Meta-X-Amz-Iv" - amzHeaderKey = "X-Amz-Meta-X-Amz-Key" - amzHeaderMatDesc = "X-Amz-Meta-X-Amz-Matdesc" -) +// Storage class header constant. +const amzStorageClass = "X-Amz-Storage-Class" + +// Website redirect location header constant +const amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location" diff --git a/core.go b/core.go index 4245fc0..4d51363 100644 --- a/core.go +++ b/core.go @@ -22,7 +22,7 @@ import ( "io" "strings" - "github.com/minio/minio-go/pkg/policy" + "github.com/minio/minio-go/pkg/encrypt" ) // Core - Inherits Client and adds new methods to expose the low level S3 APIs. @@ -50,9 +50,9 @@ func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) } // ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses -// continuationToken instead of marker to further filter the results. -func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) { - return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys) +// continuationToken instead of marker to support iteration over the results. +func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { + return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys, startAfter) } // CopyObject - copies an object from source object to destination object on server side. @@ -70,7 +70,7 @@ func (c Core) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string } // PutObject - Upload object. Uploads using single PUT call. -func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string) (ObjectInfo, error) { +func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) { opts := PutObjectOptions{} m := make(map[string]string) for k, v := range metadata { @@ -78,15 +78,20 @@ func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Ba opts.ContentEncoding = v } else if strings.ToLower(k) == "content-disposition" { opts.ContentDisposition = v + } else if strings.ToLower(k) == "content-language" { + opts.ContentLanguage = v } else if strings.ToLower(k) == "content-type" { opts.ContentType = v } else if strings.ToLower(k) == "cache-control" { opts.CacheControl = v + } else if strings.ToLower(k) == strings.ToLower(amzWebsiteRedirectLocation) { + opts.WebsiteRedirectLocation = v } else { m[k] = metadata[k] } } opts.UserMetadata = m + opts.ServerSideEncryption = sse return c.putObjectDo(context.Background(), bucket, object, data, md5Base64, sha256Hex, size, opts) } @@ -102,14 +107,8 @@ func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, de } // PutObjectPart - Upload an object part. -func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string) (ObjectPart, error) { - return c.PutObjectPartWithMetadata(bucket, object, uploadID, partID, data, size, md5Base64, sha256Hex, nil) -} - -// PutObjectPartWithMetadata - upload an object part with additional request metadata. -func (c Core) PutObjectPartWithMetadata(bucket, object, uploadID string, partID int, data io.Reader, - size int64, md5Base64, sha256Hex string, metadata map[string]string) (ObjectPart, error) { - return c.uploadPart(context.Background(), bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, metadata) +func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { + return c.uploadPart(context.Background(), bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse) } // ListObjectParts - List uploaded parts of an incomplete upload.x @@ -118,11 +117,11 @@ func (c Core) ListObjectParts(bucket, object, uploadID string, partNumberMarker } // CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. -func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) error { - _, err := c.completeMultipartUpload(context.Background(), bucket, object, uploadID, completeMultipartUpload{ +func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) (string, error) { + res, err := c.completeMultipartUpload(context.Background(), bucket, object, uploadID, completeMultipartUpload{ Parts: parts, }) - return err + return res.ETag, err } // AbortMultipartUpload - Abort an incomplete upload. @@ -131,12 +130,12 @@ func (c Core) AbortMultipartUpload(bucket, object, uploadID string) error { } // GetBucketPolicy - fetches bucket access policy for a given bucket. -func (c Core) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) { +func (c Core) GetBucketPolicy(bucket string) (string, error) { return c.getBucketPolicy(bucket) } // PutBucketPolicy - applies a new bucket access policy for a given bucket. -func (c Core) PutBucketPolicy(bucket string, bucketPolicy policy.BucketAccessPolicy) error { +func (c Core) PutBucketPolicy(bucket, bucketPolicy string) error { return c.putBucketPolicy(bucket, bucketPolicy) } diff --git a/core_test.go b/core_test.go index 8cf8104..5afdd18 100644 --- a/core_test.go +++ b/core_test.go @@ -22,7 +22,6 @@ import ( "io" "log" "os" - "reflect" "testing" "time" @@ -360,8 +359,8 @@ func TestGetBucketPolicy(t *testing.T) { t.Error("Error:", err, bucketName) } } - if !reflect.DeepEqual(bucketPolicy, emptyBucketAccessPolicy) { - t.Errorf("Bucket policy expected %#v, got %#v", emptyBucketAccessPolicy, bucketPolicy) + if bucketPolicy != "" { + t.Errorf("Bucket policy expected %#v, got %#v", "", bucketPolicy) } err = c.RemoveBucket(bucketName) @@ -411,7 +410,7 @@ func TestCoreCopyObject(t *testing.T) { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ "Content-Type": "binary/octet-stream", - }) + }, nil) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -526,7 +525,7 @@ func TestCoreCopyObjectPart(t *testing.T) { objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ "Content-Type": "binary/octet-stream", - }) + }, nil) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -566,7 +565,7 @@ func TestCoreCopyObjectPart(t *testing.T) { } // Complete the multipart upload - err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []CompletePart{fstPart, sndPart, lstPart}) + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []CompletePart{fstPart, sndPart, lstPart}) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } @@ -673,12 +672,12 @@ func TestCorePutObject(t *testing.T) { metadata := make(map[string]string) metadata["Content-Type"] = objectContentType - objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "1B2M2Y8AsgTpgAmY7PhCfg==", "", metadata) + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "1B2M2Y8AsgTpgAmY7PhCfg==", "", metadata, nil) if err == nil { t.Fatal("Error expected: error, got: nil(success)") } - objInfo, err = c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", metadata) + objInfo, err = c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", metadata, nil) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -754,7 +753,7 @@ func TestCoreGetObjectMetadata(t *testing.T) { } _, err = core.PutObject(bucketName, "my-objectname", - bytes.NewReader([]byte("hello")), 5, "", "", metadata) + bytes.NewReader([]byte("hello")), 5, "", "", metadata, nil) if err != nil { log.Fatalln(err) } diff --git a/docs/API.md b/docs/API.md index d1026ee..5778216 100644 --- a/docs/API.md +++ b/docs/API.md @@ -52,22 +52,24 @@ func main() { | Bucket operations | Object operations | Encrypted Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | | :--- | :--- | :--- | :--- | :--- | :--- | -| [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`NewSymmetricKey`](#NewSymmetricKey) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | -| [`ListBuckets`](#ListBuckets) | [`PutObject`](#PutObject) | [`NewAsymmetricKey`](#NewAsymmetricKey) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | -| [`BucketExists`](#BucketExists) | [`CopyObject`](#CopyObject) | [`GetEncryptedObject`](#GetEncryptedObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`ListBucketPolicies`](#ListBucketPolicies) | [`TraceOn`](#TraceOn) | -| [`RemoveBucket`](#RemoveBucket) | [`StatObject`](#StatObject) | [`PutEncryptedObject`](#PutEncryptedObject) | | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOff`](#TraceOff) | -| [`ListObjects`](#ListObjects) | [`RemoveObject`](#RemoveObject) | [`NewSSEInfo`](#NewSSEInfo) | | [`GetBucketNotification`](#GetBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | -| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveObjects`](#RemoveObjects) | [`FPutEncryptedObject`](#FPutEncryptedObject) | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | | -| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`ListenBucketNotification`](#ListenBucketNotification) | | -| | [`FPutObject`](#FPutObject) | | | | | -| | [`FGetObject`](#FGetObject) | | | | | -| | [`ComposeObject`](#ComposeObject) | | | | | -| | [`NewSourceInfo`](#NewSourceInfo) | | | | | -| | [`NewDestinationInfo`](#NewDestinationInfo) | | | | | -| | [`PutObjectWithContext`](#PutObjectWithContext) | | | | -| | [`GetObjectWithContext`](#GetObjectWithContext) | | | | -| | [`FPutObjectWithContext`](#FPutObjectWithContext) | | | | -| | [`FGetObjectWithContext`](#FGetObjectWithContext) | | | | +| [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`GetObject`](#GetObject) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | +| [`ListBuckets`](#ListBuckets) | [`PutObject`](#PutObject) | [`PutObject`](#PutObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | +| [`BucketExists`](#BucketExists) | [`CopyObject`](#CopyObject) | [`CopyObject`](#CopyObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | +| [`RemoveBucket`](#RemoveBucket) | [`StatObject`](#StatObject) | [`StatObject`](#StatObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | +| [`ListObjects`](#ListObjects) | [`RemoveObject`](#RemoveObject) | | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | +| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveObjects`](#RemoveObjects) | | | [`ListenBucketNotification`](#ListenBucketNotification) | | +| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | +| | [`FPutObject`](#FPutObject) | [`FPutObject`](#FPutObject) | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | +| | [`FGetObject`](#FGetObject) | [`FGetObject`](#FGetObject) | | | | +| | [`ComposeObject`](#ComposeObject) | [`ComposeObject`](#ComposeObject) | | | | +| | [`NewSourceInfo`](#NewSourceInfo) | [`NewSourceInfo`](#NewSourceInfo) | | | | +| | [`NewDestinationInfo`](#NewDestinationInfo) | [`NewDestinationInfo`](#NewDestinationInfo) | | | | +| | [`PutObjectWithContext`](#PutObjectWithContext) | [`PutObjectWithContext`](#PutObjectWithContext) | | | +| | [`GetObjectWithContext`](#GetObjectWithContext) | [`GetObjectWithContext`](#GetObjectWithContext) | | | +| | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | | +| | [`FGetObjectWithContext`](#FGetObjectWithContext) | [`FGetObjectWithContext`](#FGetObjectWithContext) | | | +| | [`RemoveObjectsWithContext`](#RemoveObjectsWithContext) | | | | +| | [`SelectObjectContent`](#SelectObjectContent) | | ## 1. Constructor @@ -86,16 +88,27 @@ __Parameters__ ### NewWithRegion(endpoint, accessKeyID, secretAccessKey string, ssl bool, region string) (*Client, error) Initializes minio client, with region configured. Unlike New(), NewWithRegion avoids bucket-location lookup operations and it is slightly faster. Use this function when your application deals with a single region. +### NewWithOptions(endpoint string, options *Options) (*Client, error) +Initializes minio client with options configured. + __Parameters__ |Param |Type |Description | |:---|:---| :---| |`endpoint` | _string_ |S3 compatible object storage endpoint | -|`accessKeyID` |_string_ |Access key for the object storage | -|`secretAccessKey` | _string_ |Secret key for the object storage | -|`ssl` | _bool_ | If 'true' API requests will be secure (HTTPS), and insecure (HTTP) otherwise | -|`region`| _string_ | Region for the object storage | +|`opts` |_minio.Options_ | Options for constructing a new client| +__minio.Options__ + +|Field | Type | Description | +|:--- |:--- | :--- | +| `opts.Creds` | _*credentials.Credentials_ | Access Credentials| +| `opts.Secure` | _bool_ | If 'true' API requests will be secure (HTTPS), and insecure (HTTP) otherwise | +| `opts.Region` | _string_ | region | +| `opts.BucketLookup` | _BucketLookupType_ | Bucket lookup type can be one of the following values | +| | | _minio.BucketLookupDNS_ | +| | | _minio.BucketLookupPath_ | +| | | _minio.BucketLookupAuto_ | ## 2. Bucket operations @@ -380,7 +393,7 @@ __minio.GetObjectOptions__ |Field | Type | Description | |:---|:---|:---| -| `opts.Materials` | _encrypt.Materials_ | Interface provided by `encrypt` package to encrypt a stream of data (For more information see https://godoc.org/github.com/minio/minio-go) | +| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | __Return Value__ @@ -513,42 +526,6 @@ if err != nil { } ``` - -### FGetEncryptedObject(bucketName, objectName, filePath string, materials encrypt.Materials) error -Identical to FGetObject operation, but decrypts an encrypted request - -__Parameters__ - - -|Param |Type |Description | -|:---|:---| :---| -|`bucketName` | _string_ |Name of the bucket | -|`objectName` | _string_ |Name of the object | -|`filePath` | _string_ |Path to download object to | -|`materials` | _encrypt.Materials_ | Interface provided by `encrypt` package to encrypt a stream of data (For more information see https://godoc.org/github.com/minio/minio-go) | - - -__Example__ - - -```go -// Generate a master symmetric key -key := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) - -// Build the CBC encryption material -cbcMaterials, err := encrypt.NewCBCSecureMaterials(key) -if err != nil { - fmt.Println(err) - return -} - -err = minioClient.FGetEncryptedObject("mybucket", "myobject", "/tmp/myobject", cbcMaterials) -if err != nil { - fmt.Println(err) - return -} -``` - ### PutObject(bucketName, objectName string, reader io.Reader, objectSize int64,opts PutObjectOptions) (n int, err error) Uploads objects that are less than 64MiB in a single PUT operation. For objects that are greater than 64MiB in size, PutObject seamlessly uploads the object as parts of 64MiB or more depending on the actual file size. The max upload size for an object is 5TB. @@ -573,9 +550,11 @@ __minio.PutObjectOptions__ | `opts.ContentType` | _string_ | Content type of object, e.g "application/text" | | `opts.ContentEncoding` | _string_ | Content encoding of object, e.g "gzip" | | `opts.ContentDisposition` | _string_ | Content disposition of object, "inline" | +| `opts.ContentLanguage` | _string_ | Content language of object, e.g "French" | | `opts.CacheControl` | _string_ | Used to specify directives for caching mechanisms in both requests and responses e.g "max-age=600"| -| `opts.EncryptMaterials` | _encrypt.Materials_ | Interface provided by `encrypt` package to encrypt a stream of data (For more information see https://godoc.org/github.com/minio/minio-go) | - +| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | +| `opts.StorageClass` | _string_ | Specify storage class for the object. Supported values for Minio server are `REDUCED_REDUNDANCY` and `STANDARD` | +| `opts.WebsiteRedirectLocation` | _string_ | Specify a redirect for the object, to another object in the same bucket or to a external URL. | __Example__ @@ -618,7 +597,7 @@ __Parameters__ |`objectName` | _string_ |Name of the object | |`reader` | _io.Reader_ |Any Go type that implements io.Reader | |`objectSize`| _int64_ | size of the object being uploaded. Pass -1 if stream size is unknown | -|`opts` | _minio.PutObjectOptions_ |Pointer to struct that allows user to set optional custom metadata, content-type, content-encoding,content-disposition and cache-control headers, pass encryption module for encrypting objects, and optionally configure number of threads for multipart put operation. | +|`opts` | _minio.PutObjectOptions_ |Pointer to struct that allows user to set optional custom metadata, content-type, content-encoding, content-disposition, content-language and cache-control headers, pass encryption module for encrypting objects, and optionally configure number of threads for multipart put operation. | __Example__ @@ -746,27 +725,27 @@ __Example__ ```go // Prepare source decryption key (here we assume same key to // decrypt all source objects.) -decKey := minio.NewSSEInfo([]byte{1, 2, 3}, "") +sseSrc := encrypt.DefaultPBKDF([]byte("password"), []byte("salt")) // Source objects to concatenate. We also specify decryption // key for each -src1 := minio.NewSourceInfo("bucket1", "object1", &decKey) +src1 := minio.NewSourceInfo("bucket1", "object1", sseSrc) src1.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") -src2 := minio.NewSourceInfo("bucket2", "object2", &decKey) +src2 := minio.NewSourceInfo("bucket2", "object2", sseSrc) src2.SetMatchETagCond("f8ef9c385918b653a31624deb84149d2") -src3 := minio.NewSourceInfo("bucket3", "object3", &decKey) +src3 := minio.NewSourceInfo("bucket3", "object3", sseSrc) src3.SetMatchETagCond("5918b653a31624deb84149d2f8ef9c38") // Create slice of sources. srcs := []minio.SourceInfo{src1, src2, src3} // Prepare destination encryption key -encKey := minio.NewSSEInfo([]byte{8, 9, 0}, "") +sseDst := encrypt.DefaultPBKDF([]byte("new-password"), []byte("new-salt")) // Create destination info -dst, err := minio.NewDestinationInfo("bucket", "object", &encKey, nil) +dst, err := minio.NewDestinationInfo("bucket", "object", sseDst, nil) if err != nil { fmt.Println(err) return @@ -792,7 +771,7 @@ __Parameters__ | :--- | :--- | :--- | | `bucket` | _string_ | Name of the source bucket | | `object` | _string_ | Name of the source object | -| `decryptSSEC` | _*minio.SSEInfo_ | Decryption info for the source object (`nil` without encryption) | +| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | __Example__ @@ -817,8 +796,8 @@ if err != nil { ```go // With decryption parameter. -decKey := minio.NewSSEInfo([]byte{1,2,3}, "") -src := minio.NewSourceInfo("bucket", "object", &decKey) +sseSrc := encrypt.DefaultPBKDF([]byte("password"), []byte("salt")) +src := minio.NewSourceInfo("bucket", "object", sseSrc) // Destination object dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) @@ -845,7 +824,7 @@ __Parameters__ | :--- | :--- | :--- | | `bucket` | _string_ | Name of the destination bucket | | `object` | _string_ | Name of the destination object | -| `encryptSSEC` | _*minio.SSEInfo_ | Encryption info for the source object (`nil` without encryption) | +| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | | | `userMeta` | _map[string]string_ | User metadata to be set on the destination. If nil, with only one source, user-metadata is copied from source. | __Example__ @@ -871,8 +850,8 @@ if err != nil { src := minio.NewSourceInfo("bucket", "object", nil) // With encryption parameter. -encKey := minio.NewSSEInfo([]byte{1,2,3}, "") -dst, err := minio.NewDestinationInfo("bucket", "object", &encKey, nil) +sseDst := encrypt.DefaultPBKDF([]byte("password"), []byte("salt")) +dst, err := minio.NewDestinationInfo("bucket", "object", sseDst, nil) if err != nil { fmt.Println(err) return @@ -900,7 +879,7 @@ __Parameters__ |`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | |`filePath` | _string_ |Path to file to be uploaded | -|`opts` | _minio.PutObjectOptions_ |Pointer to struct that allows user to set optional custom metadata, content-type, content-encoding,content-disposition and cache-control headers, pass encryption module for encrypting objects, and optionally configure number of threads for multipart put operation. | +|`opts` | _minio.PutObjectOptions_ |Pointer to struct that allows user to set optional custom metadata, content-type, content-encoding, content-disposition, content-language and cache-control headers, pass encryption module for encrypting objects, and optionally configure number of threads for multipart put operation. | __Example__ @@ -1050,306 +1029,123 @@ for rErr := range minioClient.RemoveObjects("mybucket", objectsCh) { } ``` - -### RemoveIncompleteUpload(bucketName, objectName string) error -Removes a partially uploaded object. - -__Parameters__ + +### RemoveObjectsWithContext(ctx context.Context, bucketName string, objectsCh chan string) (errorCh <-chan RemoveObjectError) +*Identical to RemoveObjects operation, but accepts a context for request cancellation.* +Parameters |Param |Type |Description | |:---|:---| :---| -|`bucketName` | _string_ |Name of the bucket | -|`objectName` | _string_ |Name of the object | - -__Example__ - - -```go -err = minioClient.RemoveIncompleteUpload("mybucket", "myobject") -if err != nil { - fmt.Println(err) - return -} -``` - -## 4. Encrypted object operations - - -### NewSymmetricKey(key []byte) *encrypt.SymmetricKey - -__Parameters__ - -|Param |Type |Description | -|:---|:---| :---| -|`key` | _string_ |Name of the bucket | +|`ctx` | _context.Context_ |Request context | +|`bucketName` | _string_ |Name of the bucket | +|`objectsCh` | _chan string_ | Channel of objects to be removed | -__Return Value__ +__Return Values__ |Param |Type |Description | |:---|:---| :---| -|`symmetricKey` | _*encrypt.SymmetricKey_ | represents a symmetric key structure which can be used to encrypt and decrypt data | +|`errorCh` | _<-chan minio.RemoveObjectError_ | Receive-only channel of errors observed during deletion. | ```go -symKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) +objectsCh := make(chan string) +ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) +defer cancel() -// Build the CBC encryption material with symmetric key. -cbcMaterials, err := encrypt.NewCBCSecureMaterials(symKey) -if err != nil { - fmt.Println(err) - return -} -fmt.Println("Successfully initialized Symmetric key CBC materials", cbcMaterials) +// Send object names that are needed to be removed to objectsCh +go func() { + defer close(objectsCh) + // List all objects from a bucket-name with a matching prefix. + for object := range minioClient.ListObjects("my-bucketname", "my-prefixname", true, nil) { + if object.Err != nil { + log.Fatalln(object.Err) + } + objectsCh <- object.Key + } +}() -object, err := minioClient.GetEncryptedObject("mybucket", "myobject", cbcMaterials) -if err != nil { - fmt.Println(err) - return +for rErr := range minioClient.RemoveObjects(ctx, "my-bucketname", objectsCh) { + fmt.Println("Error detected during deletion: ", rErr) } -defer object.Close() ``` - - -### NewAsymmetricKey(privateKey []byte, publicKey[]byte) (*encrypt.AsymmetricKey, error) - -__Parameters__ + +### SelectObjectContent(ctx context.Context, bucketName string, objectsName string, expression string, options SelectObjectOptions) *SelectResults +Parameters |Param |Type |Description | |:---|:---| :---| -|`privateKey` | _[]byte_ | Private key data | -|`publicKey` | _[]byte_ | Public key data | - +|`ctx` | _context.Context_ |Request context | +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`options` | _SelectObjectOptions_ | Query Options | -__Return Value__ +__Return Values__ |Param |Type |Description | |:---|:---| :---| -|`asymmetricKey` | _*encrypt.AsymmetricKey_ | represents an asymmetric key structure which can be used to encrypt and decrypt data | -|`err` | _error_ | Standard Error | - +|`SelectResults` | _SelectResults_ | Is an io.ReadCloser object which can be directly passed to csv.NewReader for processing output. | ```go -privateKey, err := ioutil.ReadFile("private.key") -if err != nil { - fmt.Println(err) - return -} - -publicKey, err := ioutil.ReadFile("public.key") -if err != nil { - fmt.Println(err) - return -} + // Initialize minio client object. + minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) + if err != nil { + log.Fatalln(err) + } -// Initialize the asymmetric key -asymmetricKey, err := encrypt.NewAsymmetricKey(privateKey, publicKey) -if err != nil { - fmt.Println(err) - return -} + opts := minio.SelectObjectOptions{ + Expression: "select count(*) from s3object", + ExpressionType: minio.QueryExpressionTypeSQL, + InputSerialization: minio.SelectObjectInputSerialization{ + CompressionType: minio.SelectCompressionNONE, + CSV: &minio.CSVInputOptions{ + FileHeaderInfo: minio.CSVFileHeaderInfoNone, + RecordDelimiter: "\n", + FieldDelimiter: ",", + }, + }, + OutputSerialization: minio.SelectObjectOutputSerialization{ + CSV: &minio.CSVOutputOptions{ + RecordDelimiter: "\n", + FieldDelimiter: ",", + }, + }, + } -// Build the CBC encryption material for asymmetric key. -cbcMaterials, err := encrypt.NewCBCSecureMaterials(asymmetricKey) -if err != nil { - fmt.Println(err) - return -} -fmt.Println("Successfully initialized Asymmetric key CBC materials", cbcMaterials) + reader, err := s3Client.SelectObjectContent(context.Background(), "mycsvbucket", "mycsv.csv", opts) + if err != nil { + log.Fatalln(err) + } + defer reader.Close() -object, err := minioClient.GetEncryptedObject("mybucket", "myobject", cbcMaterials) -if err != nil { - fmt.Println(err) - return -} -defer object.Close() + if _, err := io.Copy(os.Stdout, reader); err != nil { + log.Fatalln(err) + } ``` - -### GetEncryptedObject(bucketName, objectName string, encryptMaterials encrypt.Materials) (io.ReadCloser, error) - -Returns the decrypted stream of the object data based of the given encryption materials. Most of the common errors occur when reading the stream. + +### RemoveIncompleteUpload(bucketName, objectName string) error +Removes a partially uploaded object. __Parameters__ -|Param |Type |Description | -|:---|:---| :---| -|`bucketName` | _string_ | Name of the bucket | -|`objectName` | _string_ | Name of the object | -|`encryptMaterials` | _encrypt.Materials_ | Interface provided by `encrypt` package to encrypt a stream of data (For more information see https://godoc.org/github.com/minio/minio-go) | - - -__Return Value__ |Param |Type |Description | |:---|:---| :---| -|`stream` | _io.ReadCloser_ | Returns the deciphered object reader, caller should close after reading. | -|`err` | _error | Returns errors. | - - -__Example__ - - -```go -// Generate a master symmetric key -key := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) - -// Build the CBC encryption material -cbcMaterials, err := encrypt.NewCBCSecureMaterials(key) -if err != nil { - fmt.Println(err) - return -} - -object, err := minioClient.GetEncryptedObject("mybucket", "myobject", cbcMaterials) -if err != nil { - fmt.Println(err) - return -} -defer object.Close() - -localFile, err := os.Create("/tmp/local-file.jpg") -if err != nil { - fmt.Println(err) - return -} -defer localFile.Close() - -if _, err = io.Copy(localFile, object); err != nil { - fmt.Println(err) - return -} -``` - - - -### PutEncryptedObject(bucketName, objectName string, reader io.Reader, encryptMaterials encrypt.Materials) (n int, err error) -Encrypt and upload an object. - -__Parameters__ - -|Param |Type |Description | -|:---|:---| :---| -|`bucketName` | _string_ |Name of the bucket | +|`bucketName` | _string_ |Name of the bucket | |`objectName` | _string_ |Name of the object | -|`reader` | _io.Reader_ |Any Go type that implements io.Reader | -|`encryptMaterials` | _encrypt.Materials_ | Interface provided by `encrypt` package to encrypt a stream of data (For more information see https://godoc.org/github.com/minio/minio-go) | - -__Example__ - -```go -// Load a private key -privateKey, err := ioutil.ReadFile("private.key") -if err != nil { - fmt.Println(err) - return -} - -// Load a public key -publicKey, err := ioutil.ReadFile("public.key") -if err != nil { - fmt.Println(err) - return -} - -// Build an asymmetric key -key, err := encrypt.NewAsymmetricKey(privateKey, publicKey) -if err != nil { - fmt.Println(err) - return -} - -// Build the CBC encryption module -cbcMaterials, err := encrypt.NewCBCSecureMaterials(key) -if err != nil { - fmt.Println(err) - return -} - -// Open a file to upload -file, err := os.Open("my-testfile") -if err != nil { - fmt.Println(err) - return -} -defer file.Close() - -// Upload the encrypted form of the file -n, err := minioClient.PutEncryptedObject("mybucket", "myobject", file, cbcMaterials) -if err != nil { - fmt.Println(err) - return -} -fmt.Println("Successfully uploaded encrypted bytes: ", n) -``` - - -### FPutEncryptedObject(bucketName, objectName, filePath, encryptMaterials encrypt.Materials) (n int, err error) -Encrypt and upload an object from a file. - -__Parameters__ - - -|Param |Type |Description | -|:---|:---| :---| -|`bucketName` | _string_ |Name of the bucket | -|`objectName` | _string_ |Name of the object | -|`filePath` | _string_ |Path to file to be uploaded | -|`encryptMaterials` | _encrypt.Materials_ | Interface provided by `encrypt` package to encrypt a stream of data (For more information see https://godoc.org/github.com/minio/minio-go)The module that encrypts data | __Example__ ```go -// Load a private key -privateKey, err := ioutil.ReadFile("private.key") -if err != nil { - fmt.Println(err) - return -} - -// Load a public key -publicKey, err := ioutil.ReadFile("public.key") -if err != nil { - fmt.Println(err) - return -} - -// Build an asymmetric key -key, err := encrypt.NewAsymmetricKey(privateKey, publicKey) -if err != nil { - fmt.Println(err) - return -} - -// Build the CBC encryption module -cbcMaterials, err := encrypt.NewCBCSecureMaterials(key) -if err != nil { - fmt.Println(err) - return -} - -n, err := minioClient.FPutEncryptedObject("mybucket", "myobject.csv", "/tmp/otherobject.csv", cbcMaterials) +err = minioClient.RemoveIncompleteUpload("mybucket", "myobject") if err != nil { fmt.Println(err) return } -fmt.Println("Successfully uploaded encrypted bytes: ", n) ``` - - -### NewSSEInfo(key []byte, algo string) SSEInfo -Create a key object for use as encryption or decryption parameter in operations involving server-side-encryption with customer provided key (SSE-C). - -__Parameters__ - -| Param | Type | Description | -| :--- | :--- | :--- | -| `key` | _[]byte_ | Byte-slice of the raw, un-encoded binary key | -| `algo` | _string_ | Algorithm to use in encryption or decryption with the given key. Can be empty (defaults to `AES256`) | - - ## 5. Presigned operations @@ -1486,40 +1282,28 @@ fmt.Printf("%s\n", url) ## 6. Bucket policy/notification operations -### SetBucketPolicy(bucketname, objectPrefix string, policy policy.BucketPolicy) error +### SetBucketPolicy(bucketname, policy string) error Set access permissions on bucket or an object prefix. -Importing `github.com/minio/minio-go/pkg/policy` package is needed. - __Parameters__ - |Param |Type |Description | |:---|:---| :---| |`bucketName` | _string_ |Name of the bucket| -|`objectPrefix` | _string_ |Name of the object prefix| -|`policy` | _policy.BucketPolicy_ |Policy can be one of the following, | -| | | _policy.BucketPolicyNone_ | -| | | _policy.BucketPolicyReadOnly_ | -| | | _policy.BucketPolicyReadWrite_ | -| | | _policy.BucketPolicyWriteOnly_ | - +|`policy` | _string_ |Policy to be set | __Return Values__ - |Param |Type |Description | |:---|:---| :---| |`err` | _error_ |Standard Error | - __Example__ - ```go -// Sets 'mybucket' with a sub-directory 'myprefix' to be anonymously accessible for -// both read and write operations. -err = minioClient.SetBucketPolicy("mybucket", "myprefix", policy.BucketPolicyReadWrite) +policy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::my-bucketname/*"],"Sid": ""}]}` + +err = minioClient.SetBucketPolicy("my-bucketname", policy) if err != nil { fmt.Println(err) return @@ -1527,70 +1311,30 @@ if err != nil { ``` -### GetBucketPolicy(bucketName, objectPrefix string) (policy.BucketPolicy, error) +### GetBucketPolicy(bucketName) (policy string, error) Get access permissions on a bucket or a prefix. -Importing `github.com/minio/minio-go/pkg/policy` package is needed. - __Parameters__ |Param |Type |Description | |:---|:---| :---| |`bucketName` | _string_ |Name of the bucket | -|`objectPrefix` | _string_ |Prefix matching objects under the bucket | - -__Return Values__ - - -|Param |Type |Description | -|:---|:---| :---| -|`bucketPolicy` | _policy.BucketPolicy_ |string that contains: `none`, `readonly`, `readwrite`, or `writeonly` | -|`err` | _error_ |Standard Error | - -__Example__ - - -```go -bucketPolicy, err := minioClient.GetBucketPolicy("mybucket", "") -if err != nil { - fmt.Println(err) - return -} -fmt.Println("Access permissions for mybucket is", bucketPolicy) -``` - - -### ListBucketPolicies(bucketName, objectPrefix string) (map[string]BucketPolicy, error) -Get access permissions rules associated to the specified bucket and prefix. - -__Parameters__ - - -|Param |Type |Description | -|:---|:---| :---| -|`bucketName` | _string_ |Name of the bucket | -|`objectPrefix` | _string_ |Prefix matching objects under the bucket | __Return Values__ |Param |Type |Description | |:---|:---| :---| -|`bucketPolicies` | _map[string]minio.BucketPolicy_ |Map of object resource paths and their permissions | +|`policy` | _string_ |Policy returned from the server | |`err` | _error_ |Standard Error | __Example__ - ```go -bucketPolicies, err := minioClient.ListBucketPolicies("mybucket", "") +policy, err := minioClient.GetBucketPolicy("my-bucketname") if err != nil { - fmt.Println(err) - return -} -for resource, permission := range bucketPolicies { - fmt.Println(resource, " => ", permission) + log.Fatalln(err) } ``` @@ -1755,6 +1499,72 @@ for notificationInfo := range minioClient.ListenBucketNotification("mybucket", " } ``` + +### SetBucketLifecycle(bucketname, lifecycle string) error +Set lifecycle on bucket or an object prefix. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket| +|`lifecycle` | _string_ |Lifecycle to be set | + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + +```go +lifecycle := ` + + expire-bucket + + Enabled + + 365 + + +` + +err = minioClient.SetBucketLifecycle("my-bucketname", lifecycle) +if err != nil { + fmt.Println(err) + return +} +``` + + +### GetBucketLifecycle(bucketName) (lifecycle string, error) +Get lifecycle on a bucket or a prefix. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`lifecycle` | _string_ |Lifecycle returned from the server | +|`err` | _error_ |Standard Error | + +__Example__ + +```go +lifecycle, err := minioClient.GetBucketLifecycle("my-bucketname") +if err != nil { + log.Fatalln(err) +} +``` + ## 7. Client custom settings diff --git a/docs/zh_CN/API.md b/docs/zh_CN/API.md index d20ca10..ca80586 100644 --- a/docs/zh_CN/API.md +++ b/docs/zh_CN/API.md @@ -54,11 +54,11 @@ func main() { | :--- | :--- | :--- | :--- | :--- | :--- | | [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`NewSymmetricKey`](#NewSymmetricKey) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | | [`ListBuckets`](#ListBuckets) | [`PutObject`](#PutObject) | [`NewAsymmetricKey`](#NewAsymmetricKey) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | -| [`BucketExists`](#BucketExists) | [`CopyObject`](#CopyObject) | [`GetEncryptedObject`](#GetEncryptedObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`ListBucketPolicies`](#ListBucketPolicies) | [`TraceOn`](#TraceOn) | -| [`RemoveBucket`](#RemoveBucket) | [`StatObject`](#StatObject) | [`PutEncryptedObject`](#PutEncryptedObject) | | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOff`](#TraceOff) | -| [`ListObjects`](#ListObjects) | [`RemoveObject`](#RemoveObject) | [`NewSSEInfo`](#NewSSEInfo) | | [`GetBucketNotification`](#GetBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | -| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveObjects`](#RemoveObjects) | [`FPutEncryptedObject`](#FPutEncryptedObject) | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | | -| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`ListenBucketNotification`](#ListenBucketNotification) | | +| [`BucketExists`](#BucketExists) | [`CopyObject`](#CopyObject) | [`GetEncryptedObject`](#GetEncryptedObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | +| [`RemoveBucket`](#RemoveBucket) | [`StatObject`](#StatObject) | [`PutEncryptedObject`](#PutEncryptedObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | +| [`ListObjects`](#ListObjects) | [`RemoveObject`](#RemoveObject) | [`NewSSEInfo`](#NewSSEInfo) | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | +| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveObjects`](#RemoveObjects) | [`FPutEncryptedObject`](#FPutEncryptedObject) | | [`ListenBucketNotification`](#ListenBucketNotification) | | +| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | | | | | [`FPutObject`](#FPutObject) | | | | | | | [`FGetObject`](#FGetObject) | | | | | | | [`ComposeObject`](#ComposeObject) | | | | | @@ -1560,40 +1560,6 @@ if err != nil { fmt.Println("Access permissions for mybucket is", bucketPolicy) ``` - -### ListBucketPolicies(bucketName, objectPrefix string) (map[string]BucketPolicy, error) -获取指定的存储桶和前缀的访问策略。 - -__参数__ - - -|参数 |类型 |描述 | -|:---|:---| :---| -|`bucketName` | _string_ |存储桶名称 | -|`objectPrefix` | _string_ |该存储桶下的对象前缀 | - -__返回值__ - - -|参数 |类型 |描述 | -|:---|:---| :---| -|`bucketPolicies` | _map[string]minio.BucketPolicy_ |对象以及它们的权限的Map | -|`err` | _error_ |标准Error | - -__示例__ - - -```go -bucketPolicies, err := minioClient.ListBucketPolicies("mybucket", "") -if err != nil { - fmt.Println(err) - return -} -for resource, permission := range bucketPolicies { - fmt.Println(resource, " => ", permission) -} -``` - ### GetBucketNotification(bucketName string) (BucketNotification, error) 获取存储桶的通知配置 diff --git a/examples/s3/fputencrypted-object.go b/examples/s3/fputencrypted-object.go index 96eec7e..5da9f9d 100644 --- a/examples/s3/fputencrypted-object.go +++ b/examples/s3/fputencrypted-object.go @@ -22,8 +22,9 @@ package main import ( "log" - "github.com/minio/minio-go" "github.com/minio/minio-go/pkg/encrypt" + + "github.com/minio/minio-go" ) func main() { @@ -40,38 +41,16 @@ func main() { log.Fatalln(err) } - // Specify a local file that we will upload - filePath := "my-testfile" - - //// Build an asymmetric key from private and public files - // - // privateKey, err := ioutil.ReadFile("private.key") - // if err != nil { - // t.Fatal(err) - // } - // - // publicKey, err := ioutil.ReadFile("public.key") - // if err != nil { - // t.Fatal(err) - // } - // - // asymmetricKey, err := NewAsymmetricKey(privateKey, publicKey) - // if err != nil { - // t.Fatal(err) - // } - //// + filePath := "my-testfile" // Specify a local file that we will upload + bucketname := "my-bucketname" // Specify a bucket name - the bucket must already exist + objectName := "my-objectname" // Specify a object name + password := "correct horse battery staple" // Specify your password. DO NOT USE THIS ONE - USE YOUR OWN. - // Build a symmetric key - symmetricKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) - - // Build encryption materials which will encrypt uploaded data - cbcMaterials, err := encrypt.NewCBCSecureMaterials(symmetricKey) - if err != nil { - log.Fatalln(err) - } + // New SSE-C where the cryptographic key is derived from a password and the objectname + bucketname as salt + encryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketname+objectName)) // Encrypt file content and upload to the server - n, err := s3Client.FPutEncryptedObject("my-bucketname", "my-objectname", filePath, cbcMaterials) + n, err := s3Client.FPutObject(bucketname, objectName, filePath, minio.PutObjectOptions{ServerSideEncryption: encryption}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/get-encrypted-object.go b/examples/s3/get-encrypted-object.go index 9783beb..62a06d5 100644 --- a/examples/s3/get-encrypted-object.go +++ b/examples/s3/get-encrypted-object.go @@ -42,35 +42,15 @@ func main() { log.Fatalln(err) } - //// Build an asymmetric key from private and public files - // - // privateKey, err := ioutil.ReadFile("private.key") - // if err != nil { - // t.Fatal(err) - // } - // - // publicKey, err := ioutil.ReadFile("public.key") - // if err != nil { - // t.Fatal(err) - // } - // - // asymmetricKey, err := NewAsymmetricKey(privateKey, publicKey) - // if err != nil { - // t.Fatal(err) - // } - //// + bucketname := "my-bucketname" // Specify a bucket name - the bucket must already exist + objectName := "my-objectname" // Specify a object name - the object must already exist + password := "correct horse battery staple" // Specify your password. DO NOT USE THIS ONE - USE YOUR OWN. - // Build a symmetric key - symmetricKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) + // New SSE-C where the cryptographic key is derived from a password and the objectname + bucketname as salt + encryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketname+objectName)) - // Build encryption materials which will encrypt uploaded data - cbcMaterials, err := encrypt.NewCBCSecureMaterials(symmetricKey) - if err != nil { - log.Fatalln(err) - } - - // Get a deciphered data from the server, deciphering is assured by cbcMaterials - reader, err := s3Client.GetEncryptedObject("my-bucketname", "my-objectname", cbcMaterials) + // Get the encrypted object + reader, err := s3Client.GetObject(bucketname, objectName, minio.GetObjectOptions{ServerSideEncryption: encryption}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/getbucketlifecycle.go b/examples/s3/getbucketlifecycle.go new file mode 100644 index 0000000..2e3ef41 --- /dev/null +++ b/examples/s3/getbucketlifecycle.go @@ -0,0 +1,65 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "io" + "log" + "os" + "strings" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Get bucket lifecycle from S3 + lifecycle, err := s3Client.GetBucketLifecycle("my-bucketname") + if err != nil { + log.Fatalln(err) + } + + // Create lifecycle file + localLifecycleFile, err := os.Create("lifecycle.json") + if err != nil { + log.Fatalln(err) + } + defer localLifecycleFile.Close() + + lifecycleReader := strings.NewReader(lifecycle) + + if _, err := io.Copy(localLifecycleFile, lifecycleReader); err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/getbucketpolicy.go b/examples/s3/getbucketpolicy.go index f9ac89b..e5b5940 100644 --- a/examples/s3/getbucketpolicy.go +++ b/examples/s3/getbucketpolicy.go @@ -20,7 +20,10 @@ package main import ( + "io" "log" + "os" + "strings" "github.com/minio/minio-go" ) @@ -41,16 +44,21 @@ func main() { // s3Client.TraceOn(os.Stderr) - // Fetch the policy at 'my-objectprefix'. - policy, err := s3Client.GetBucketPolicy("my-bucketname", "my-objectprefix") + policy, err := s3Client.GetBucketPolicy("my-bucketname") if err != nil { log.Fatalln(err) } - // Description of policy output. - // "none" - The specified bucket does not have a bucket policy. - // "readonly" - Read only operations are allowed. - // "writeonly" - Write only operations are allowed. - // "readwrite" - both read and write operations are allowed, the bucket is public. - log.Println("Success - ", policy) + // Create policy file + localFile, err := os.Create("policy.json") + if err != nil { + log.Fatalln(err) + } + defer localFile.Close() + + policyReader := strings.NewReader(policy) + + if _, err := io.Copy(localFile, policyReader); err != nil { + log.Fatalln(err) + } } diff --git a/examples/s3/getobject-client-encryption.go b/examples/s3/getobject-client-encryption.go new file mode 100644 index 0000000..6b06073 --- /dev/null +++ b/examples/s3/getobject-client-encryption.go @@ -0,0 +1,66 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "os" + "path" + + "github.com/minio/minio-go" + "github.com/minio/sio" + "golang.org/x/crypto/argon2" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + obj, err := s3Client.GetObject("my-bucketname", "my-objectname", minio.GetObjectOptions{}) + if err != nil { + log.Fatalln(err) + } + + localFile, err := os.Create("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer localFile.Close() + + password := []byte("myfavoritepassword") // Change as per your needs. + salt := []byte(path.Join("my-bucketname", "my-objectname")) // Change as per your needs. + _, err = sio.Decrypt(localFile, obj, sio.Config{ + Key: argon2.IDKey(password, salt, 1, 64*1024, 4, 32), // generate a 256 bit long key. + }) + if err != nil { + log.Fatalln(err) + } + log.Println("Successfully decrypted 'my-objectname' to local file 'my-testfile'") +} diff --git a/examples/s3/getobjectacl.go b/examples/s3/getobjectacl.go new file mode 100644 index 0000000..f2bbd95 --- /dev/null +++ b/examples/s3/getobjectacl.go @@ -0,0 +1,53 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018-2019 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + + minio "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + + objectInfo, err := s3Client.GetObjectACL("my-bucketname", "my-objectname") + if err != nil { + log.Fatalln(err) + } + + //print all value header (acl, metadata, standard header value...) + for k, v := range objectInfo.Metadata { + fmt.Println("key:", k) + fmt.Printf(" - value: %v\n", v) + } +} diff --git a/examples/s3/listbucketpolicies.go b/examples/s3/listbucketpolicies.go deleted file mode 100644 index 43edd0c..0000000 --- a/examples/s3/listbucketpolicies.go +++ /dev/null @@ -1,57 +0,0 @@ -// +build ignore - -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "log" - - "github.com/minio/minio-go" -) - -func main() { - // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are - // dummy values, please replace them with original values. - - // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. - // This boolean value is the last argument for New(). - - // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically - // determined based on the Endpoint value. - s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) - if err != nil { - log.Fatalln(err) - } - - // s3Client.TraceOn(os.Stderr) - - // Fetch the policy at 'my-objectprefix'. - policies, err := s3Client.ListBucketPolicies("my-bucketname", "my-objectprefix") - if err != nil { - log.Fatalln(err) - } - - // ListBucketPolicies returns a map of objects policy rules and their associated permissions - // e.g. mybucket/downloadfolder/* => readonly - // mybucket/shared/* => readwrite - - for resource, permission := range policies { - log.Println(resource, " => ", permission) - } -} diff --git a/examples/s3/put-encrypted-object.go b/examples/s3/put-encrypted-object.go index cdf09ac..48b9367 100644 --- a/examples/s3/put-encrypted-object.go +++ b/examples/s3/put-encrypted-object.go @@ -41,42 +41,30 @@ func main() { log.Fatalln(err) } + filePath := "my-testfile" // Specify a local file that we will upload + // Open a local file that we will upload - file, err := os.Open("my-testfile") + file, err := os.Open(filePath) if err != nil { log.Fatalln(err) } defer file.Close() - //// Build an asymmetric key from private and public files - // - // privateKey, err := ioutil.ReadFile("private.key") - // if err != nil { - // t.Fatal(err) - // } - // - // publicKey, err := ioutil.ReadFile("public.key") - // if err != nil { - // t.Fatal(err) - // } - // - // asymmetricKey, err := NewAsymmetricKey(privateKey, publicKey) - // if err != nil { - // t.Fatal(err) - // } - //// - - // Build a symmetric key - symmetricKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) - - // Build encryption materials which will encrypt uploaded data - cbcMaterials, err := encrypt.NewCBCSecureMaterials(symmetricKey) + // Get file stats. + fstat, err := file.Stat() if err != nil { log.Fatalln(err) } + bucketname := "my-bucketname" // Specify a bucket name - the bucket must already exist + objectName := "my-objectname" // Specify a object name + password := "correct horse battery staple" // Specify your password. DO NOT USE THIS ONE - USE YOUR OWN. + + // New SSE-C where the cryptographic key is derived from a password and the objectname + bucketname as salt + encryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketname+objectName)) + // Encrypt file content and upload to the server - n, err := s3Client.PutEncryptedObject("my-bucketname", "my-objectname", file, cbcMaterials) + n, err := s3Client.PutObject(bucketname, objectName, file, fstat.Size(), minio.PutObjectOptions{ServerSideEncryption: encryption}) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/putobject-client-encryption.go b/examples/s3/putobject-client-encryption.go new file mode 100644 index 0000000..77d83b4 --- /dev/null +++ b/examples/s3/putobject-client-encryption.go @@ -0,0 +1,75 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "os" + "path" + + "github.com/minio/minio-go" + "github.com/minio/sio" + "golang.org/x/crypto/argon2" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + object, err := os.Open("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer object.Close() + objectStat, err := object.Stat() + if err != nil { + log.Fatalln(err) + } + + password := []byte("myfavoritepassword") // Change as per your needs. + salt := []byte(path.Join("my-bucketname", "my-objectname")) // Change as per your needs. + encrypted, err := sio.EncryptReader(object, sio.Config{ + // generate a 256 bit long key. + Key: argon2.IDKey(password, salt, 1, 64*1024, 4, 32), + }) + if err != nil { + log.Fatalln(err) + } + + encSize, err := sio.EncryptedSize(uint64(objectStat.Size())) + if err != nil { + log.Fatalln(err) + } + _, err = s3Client.PutObject("my-bucketname", "my-objectname", encrypted, int64(encSize), minio.PutObjectOptions{}) + if err != nil { + log.Fatalln(err) + } + log.Println("Successfully encrypted 'my-objectname'") +} diff --git a/examples/s3/putobject-getobject-sse.go b/examples/s3/putobject-getobject-sse.go index 3d3b2fd..4e459b5 100644 --- a/examples/s3/putobject-getobject-sse.go +++ b/examples/s3/putobject-getobject-sse.go @@ -21,11 +21,11 @@ package main import ( "bytes" - "crypto/md5" - "encoding/base64" "io/ioutil" "log" + "github.com/minio/minio-go/pkg/encrypt" + minio "github.com/minio/minio-go" ) @@ -40,38 +40,19 @@ func main() { log.Fatalln(err) } - content := bytes.NewReader([]byte("Hello again")) - key := []byte("32byteslongsecretkeymustprovided") - h := md5.New() - h.Write(key) - encryptionKey := base64.StdEncoding.EncodeToString(key) - encryptionKeyMD5 := base64.StdEncoding.EncodeToString(h.Sum(nil)) - - // Amazon S3 does not store the encryption key you provide. - // Instead S3 stores a randomly salted HMAC value of the - // encryption key in order to validate future requests. - // The salted HMAC value cannot be used to derive the value - // of the encryption key or to decrypt the contents of the - // encrypted object. That means, if you lose the encryption - // key, you lose the object. - var metadata = map[string]string{ - "x-amz-server-side-encryption-customer-algorithm": "AES256", - "x-amz-server-side-encryption-customer-key": encryptionKey, - "x-amz-server-side-encryption-customer-key-MD5": encryptionKeyMD5, - } + bucketName := "my-bucket" + objectName := "my-encrypted-object" + object := []byte("Hello again") - // minioClient.TraceOn(os.Stderr) // Enable to debug. - _, err = minioClient.PutObject("mybucket", "my-encrypted-object.txt", content, 11, minio.PutObjectOptions{UserMetadata: metadata}) + encryption := encrypt.DefaultPBKDF([]byte("my secret password"), []byte(bucketName+objectName)) + _, err = minioClient.PutObject(bucketName, objectName, bytes.NewReader(object), int64(len(object)), minio.PutObjectOptions{ + ServerSideEncryption: encryption, + }) if err != nil { log.Fatalln(err) } - opts := minio.GetObjectOptions{} - for k, v := range metadata { - opts.Set(k, v) - } - coreClient := minio.Core{minioClient} - reader, _, err := coreClient.GetObject("mybucket", "my-encrypted-object.txt", opts) + reader, err := minioClient.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: encryption}) if err != nil { log.Fatalln(err) } @@ -81,7 +62,7 @@ func main() { if err != nil { log.Fatalln(err) } - if !bytes.Equal(decBytes, []byte("Hello again")) { - log.Fatalln("Expected \"Hello, world\", got %s", string(decBytes)) + if !bytes.Equal(decBytes, object) { + log.Fatalln("Expected %s, got %s", string(object), string(decBytes)) } } diff --git a/examples/s3/removeobjects.go b/examples/s3/removeobjects.go index b912bc8..a581134 100644 --- a/examples/s3/removeobjects.go +++ b/examples/s3/removeobjects.go @@ -44,6 +44,12 @@ func main() { // Send object names that are needed to be removed to objectsCh go func() { defer close(objectsCh) + + doneCh := make(chan struct{}) + + // Indicate to our routine to exit cleanly upon return. + defer close(doneCh) + // List all objects from a bucket-name with a matching prefix. for object := range s3Client.ListObjects("my-bucketname", "my-prefixname", true, doneCh) { if object.Err != nil { diff --git a/examples/s3/selectobject.go b/examples/s3/selectobject.go new file mode 100644 index 0000000..e23ccf8 --- /dev/null +++ b/examples/s3/selectobject.go @@ -0,0 +1,73 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "io" + "log" + "os" + + minio "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + + opts := minio.SelectObjectOptions{ + Expression: "select count(*) from s3object", + ExpressionType: minio.QueryExpressionTypeSQL, + InputSerialization: minio.SelectObjectInputSerialization{ + CompressionType: minio.SelectCompressionNONE, + CSV: &minio.CSVInputOptions{ + FileHeaderInfo: minio.CSVFileHeaderInfoNone, + RecordDelimiter: "\n", + FieldDelimiter: ",", + }, + }, + OutputSerialization: minio.SelectObjectOutputSerialization{ + CSV: &minio.CSVOutputOptions{ + RecordDelimiter: "\n", + FieldDelimiter: ",", + }, + }, + } + + reader, err := s3Client.SelectObjectContent(context.Background(), "mycsvbucket", "mycsv.csv", opts) + if err != nil { + log.Fatalln(err) + } + defer reader.Close() + + if _, err := io.Copy(os.Stdout, reader); err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/setbucketlifecycle.go b/examples/s3/setbucketlifecycle.go new file mode 100644 index 0000000..7eaa946 --- /dev/null +++ b/examples/s3/setbucketlifecycle.go @@ -0,0 +1,50 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Set lifecycle on a bucket + lifecycle := `expire-bucketEnabled365` + err = s3Client.SetBucketLifecycle("my-bucketname", lifecycle) + if err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/setbucketpolicy.go b/examples/s3/setbucketpolicy.go index c81fb50..bc42da8 100644 --- a/examples/s3/setbucketpolicy.go +++ b/examples/s3/setbucketpolicy.go @@ -23,7 +23,6 @@ import ( "log" "github.com/minio/minio-go" - "github.com/minio/minio-go/pkg/policy" ) func main() { @@ -42,14 +41,11 @@ func main() { // s3Client.TraceOn(os.Stderr) - // Description of policy input. - // policy.BucketPolicyNone - Remove any previously applied bucket policy at a prefix. - // policy.BucketPolicyReadOnly - Set read-only operations at a prefix. - // policy.BucketPolicyWriteOnly - Set write-only operations at a prefix. - // policy.BucketPolicyReadWrite - Set read-write operations at a prefix. - err = s3Client.SetBucketPolicy("my-bucketname", "my-objectprefix", policy.BucketPolicyReadWrite) + // Create policy + policy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::my-bucketname/*"],"Sid": ""}]}` + + err = s3Client.SetBucketPolicy("my-bucketname", policy) if err != nil { log.Fatalln(err) } - log.Println("Success") } diff --git a/functional_tests.go b/functional_tests.go index 2e0d1e7..62be0e9 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -22,7 +22,6 @@ package main import ( "bytes" "context" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -45,7 +44,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/minio/minio-go/pkg/encrypt" - "github.com/minio/minio-go/pkg/policy" ) const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" @@ -268,8 +266,8 @@ var dataFileMap = map[string]int{ "datafile-65-MB": 65 * humanize.MiByte, } -func isQuickMode() bool { - return os.Getenv("MODE") == "quick" +func isFullMode() bool { + return os.Getenv("MINT_MODE") == "full" } func getFuncName() string { @@ -571,7 +569,7 @@ func testPutObjectReadAt() { logError(testName, function, args, startTime, "", fmt.Sprintf("Number of bytes in stat does not match, expected %d got %d", bufSize, st.Size), err) return } - if st.ContentType != objectContentType { + if st.ContentType != objectContentType && st.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "Content types don't match", err) return } @@ -605,8 +603,8 @@ func testPutObjectWithMetadata() { "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", } - if isQuickMode() { - ignoredLog(testName, function, args, startTime, "Skipping functional tests for short runs").Info() + if !isFullMode() { + ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info() return } @@ -685,7 +683,7 @@ func testPutObjectWithMetadata() { logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) return } - if st.ContentType != customContentType { + if st.ContentType != customContentType && st.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "ContentType does not match, expected "+customContentType+" got "+st.ContentType, err) return } @@ -707,13 +705,12 @@ func testPutObjectWithMetadata() { successLogger(testName, function, args, startTime).Info() } -// Test put object with streaming signature. -func testPutObjectStreaming() { +func testPutObjectWithContentLanguage() { // initialize logging params objectName := "test-object" startTime := time.Now() testName := getFuncName() - function := "PutObject(bucketName, objectName, reader,size,opts)" + function := "PutObject(bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": objectName, @@ -752,21 +749,29 @@ func testPutObjectStreaming() { return } - // Upload an object. - sizes := []int64{0, 64*1024 - 1, 64 * 1024} + data := bytes.Repeat([]byte("a"), int(0)) + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{ + ContentLanguage: "en", + }) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } - for _, size := range sizes { - data := bytes.Repeat([]byte("a"), int(size)) - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(size), minio.PutObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) - return - } + if n != 0 { + logError(testName, function, args, startTime, "", "Expected upload object '0' doesn't match with PutObject return value", err) + return + } - if n != size { - logError(testName, function, args, startTime, "", "Expected upload object size doesn't match with PutObjectStreaming return value", err) - return - } + objInfo, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + + if objInfo.Metadata.Get("Content-Language") != "en" { + logError(testName, function, args, startTime, "", "Expected content-language 'en' doesn't match with StatObject return value", err) + return } // Delete all objects and buckets @@ -778,23 +783,25 @@ func testPutObjectStreaming() { successLogger(testName, function, args, startTime).Info() } -// Test listing partially uploaded objects. -func testListPartiallyUploaded() { +// Test put object with streaming signature. +func testPutObjectStreaming() { // initialize logging params + objectName := "test-object" startTime := time.Now() testName := getFuncName() - function := "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" + function := "PutObject(bucketName, objectName, reader,size,opts)" args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "isRecursive": "", + "bucketName": "", + "objectName": objectName, + "size": -1, + "opts": "", } // Seed random based on current time. rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.New( + c, err := minio.NewV4( os.Getenv(serverEndpoint), os.Getenv(accessKey), os.Getenv(secretKey), @@ -805,16 +812,15 @@ func testListPartiallyUploaded() { return } + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + // Set user agent. c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName - // Make a new bucket. err = c.MakeBucket(bucketName, "us-east-1") if err != nil { @@ -822,46 +828,19 @@ func testListPartiallyUploaded() { return } - bufSize := dataFileMap["datafile-65-MB"] - r := bytes.NewReader(bytes.Repeat([]byte("0"), bufSize*2)) + // Upload an object. + sizes := []int64{0, 64*1024 - 1, 64 * 1024} - reader, writer := io.Pipe() - go func() { - i := 0 - for i < 25 { - _, cerr := io.CopyN(writer, r, (int64(bufSize)*2)/25) - if cerr != nil { - logError(testName, function, args, startTime, "", "Copy failed", err) - return - } - i++ - r.Seek(0, 0) + for _, size := range sizes { + data := bytes.Repeat([]byte("a"), int(size)) + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(size), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) + return } - writer.CloseWithError(errors.New("proactively closed to be verified later")) - }() - - objectName := bucketName + "-resumable" - args["objectName"] = objectName - _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize*2), minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err == nil { - logError(testName, function, args, startTime, "", "PutObject should fail", err) - return - } - if !strings.Contains(err.Error(), "proactively closed to be verified later") { - logError(testName, function, args, startTime, "", "String not found in PutObject output", err) - return - } - - doneCh := make(chan struct{}) - defer close(doneCh) - isRecursive := true - args["isRecursive"] = isRecursive - - multiPartObjectCh := c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) - for multiPartObject := range multiPartObjectCh { - if multiPartObject.Err != nil { - logError(testName, function, args, startTime, "", "Multipart object error", multiPartObject.Err) + if n != size { + logError(testName, function, args, startTime, "", "Expected upload object size doesn't match with PutObjectStreaming return value", err) return } } @@ -1101,27 +1080,26 @@ func testGetObjectClosedTwice() { successLogger(testName, function, args, startTime).Info() } -// Test removing multiple objects with Remove API -func testRemoveMultipleObjects() { - // initialize logging params +// Test RemoveObjectsWithContext request context cancels after timeout +func testRemoveObjectsWithContext() { + // Initialize logging params. startTime := time.Now() testName := getFuncName() - function := "RemoveObjects(bucketName, objectsCh)" + function := "RemoveObjectsWithContext(ctx, bucketName, objectsCh)" args := map[string]interface{}{ "bucketName": "", } - // Seed random based on current time. + // Seed random based on current tie. rand.Seed(time.Now().Unix()) - // Instantiate new minio client object. + // Instantiate new minio client. c, err := minio.New( os.Getenv(serverEndpoint), os.Getenv(accessKey), os.Getenv(secretKey), mustParseBool(os.Getenv(enableHTTPS)), ) - if err != nil { logError(testName, function, args, startTime, "", "Minio client object creation failed", err) return @@ -1129,7 +1107,6 @@ func testRemoveMultipleObjects() { // Set user agent. c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - // Enable tracing, write to stdout. // c.TraceOn(os.Stderr) @@ -1141,19 +1118,16 @@ func testRemoveMultipleObjects() { err = c.MakeBucket(bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return } + // Generate put data. r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) - // Multi remove of 1100 objects - nrObjects := 200 - + // Multi remove of 20 objects. + nrObjects := 20 objectsCh := make(chan string) - go func() { defer close(objectsCh) - // Upload objects and send them to objectsCh for i := 0; i < nrObjects; i++ { objectName := "sample" + strconv.Itoa(i) + ".txt" _, err = c.PutObject(bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) @@ -1164,35 +1138,52 @@ func testRemoveMultipleObjects() { objectsCh <- objectName } }() + // Set context to cancel in 1 nanosecond. + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + args["ctx"] = ctx + defer cancel() - // Call RemoveObjects API - errorCh := c.RemoveObjects(bucketName, objectsCh) - - // Check if errorCh doesn't receive any error + // Call RemoveObjectsWithContext API with short timeout. + errorCh := c.RemoveObjectsWithContext(ctx, bucketName, objectsCh) + // Check for error. + select { + case r := <-errorCh: + if r.Err == nil { + logError(testName, function, args, startTime, "", "RemoveObjectsWithContext should fail on short timeout", err) + return + } + } + // Set context with longer timeout. + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) + args["ctx"] = ctx + defer cancel() + // Perform RemoveObjectsWithContext with the longer timeout. Expect the removals to succeed. + errorCh = c.RemoveObjectsWithContext(ctx, bucketName, objectsCh) select { case r, more := <-errorCh: - if more { + if more || r.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error", r.Err) return } } - // Delete all objects and buckets + // Delete all objects and buckets. if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) return } - successLogger(testName, function, args, startTime).Info() } -// Tests removing partially uploaded objects. -func testRemovePartiallyUploaded() { +// Test removing multiple objects with Remove API +func testRemoveMultipleObjects() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "RemoveIncompleteUpload(bucketName, objectName)" - args := map[string]interface{}{} + function := "RemoveObjects(bucketName, objectsCh)" + args := map[string]interface{}{ + "bucketName": "", + } // Seed random based on current time. rand.Seed(time.Now().Unix()) @@ -1204,6 +1195,7 @@ func testRemovePartiallyUploaded() { os.Getenv(secretKey), mustParseBool(os.Getenv(enableHTTPS)), ) + if err != nil { logError(testName, function, args, startTime, "", "Minio client object creation failed", err) return @@ -1226,40 +1218,39 @@ func testRemovePartiallyUploaded() { return } - r := bytes.NewReader(bytes.Repeat([]byte("a"), 128*1024)) + r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) + + // Multi remove of 1100 objects + nrObjects := 200 + + objectsCh := make(chan string) - reader, writer := io.Pipe() go func() { - i := 0 - for i < 25 { - _, cerr := io.CopyN(writer, r, 128*1024) - if cerr != nil { - logError(testName, function, args, startTime, "", "Copy failed", err) - return + defer close(objectsCh) + // Upload objects and send them to objectsCh + for i := 0; i < nrObjects; i++ { + objectName := "sample" + strconv.Itoa(i) + ".txt" + _, err = c.PutObject(bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + continue } - i++ - r.Seek(0, 0) + objectsCh <- objectName } - writer.CloseWithError(errors.New("proactively closed to be verified later")) }() - objectName := bucketName + "-resumable" - args["objectName"] = objectName + // Call RemoveObjects API + errorCh := c.RemoveObjects(bucketName, objectsCh) - _, err = c.PutObject(bucketName, objectName, reader, 128*1024, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err == nil { - logError(testName, function, args, startTime, "", "PutObject should fail", err) - return - } - if !strings.Contains(err.Error(), "proactively closed to be verified later") { - logError(testName, function, args, startTime, "", "String not found", err) - return - } - err = c.RemoveIncompleteUpload(bucketName, objectName) - if err != nil { - logError(testName, function, args, startTime, "", "RemoveIncompleteUpload failed", err) - return + // Check if errorCh doesn't receive any error + select { + case r, more := <-errorCh: + if more { + logError(testName, function, args, startTime, "", "Unexpected error", r.Err) + return + } } + // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) @@ -1368,7 +1359,7 @@ func testFPutObjectMultipart() { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(totalSize))+" got "+string(objInfo.Size), err) return } - if objInfo.ContentType != objectContentType { + if objInfo.ContentType != objectContentType && objInfo.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "ContentType doesn't match", err) return } @@ -1508,6 +1499,7 @@ func testFPutObject() { // Perform FPutObject with no contentType provided (Expecting application/x-gtar) args["objectName"] = objectName + "-GTar" + args["opts"] = minio.PutObjectOptions{} n, err = c.FPutObject(bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) @@ -1539,7 +1531,7 @@ func testFPutObject() { return } if rOctet.ContentType != "application/octet-stream" { - logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rStandard.ContentType, err) + logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rOctet.ContentType, err) return } @@ -1550,8 +1542,8 @@ func testFPutObject() { logError(testName, function, args, startTime, "", "StatObject failed", err) return } - if rGTar.ContentType != "application/x-gtar" { - logError(testName, function, args, startTime, "", "ContentType does not match, expected application/x-gtar, got "+rStandard.ContentType, err) + if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "ContentType does not match, expected application/x-gtar or application/octet-stream, got "+rGTar.ContentType, err) return } @@ -1912,6 +1904,14 @@ func testGetObjectReadSeekFunctional() { return } + defer func() { + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + }() + // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] var reader = getDataReader("datafile-33-kB") @@ -1938,14 +1938,6 @@ func testGetObjectReadSeekFunctional() { return } - defer func() { - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - }() - // Read the data back r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) if err != nil { @@ -2127,7 +2119,7 @@ func testGetObjectReadAtFunctional() { buf3 := make([]byte, 512) buf4 := make([]byte, 512) - // Test readAt before stat is called. + // Test readAt before stat is called such that objectInfo doesn't change. m, err := r.ReadAt(buf1, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) @@ -2167,6 +2159,7 @@ func testGetObjectReadAtFunctional() { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } + offset += 512 m, err = r.ReadAt(buf3, offset) if err != nil { @@ -2278,7 +2271,8 @@ func testPresignedPostPolicy() { defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "") + // Azure requires the key to not start with a number + metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user") metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") buf, err := ioutil.ReadAll(reader) @@ -2411,9 +2405,10 @@ func testPresignedPostPolicy() { } expectedLocation := scheme + os.Getenv(serverEndpoint) + "/" + bucketName + "/" + objectName + expectedLocationBucketDNS := scheme + bucketName + "." + os.Getenv(serverEndpoint) + "/" + objectName if val, ok := res.Header["Location"]; ok { - if val[0] != expectedLocation { + if val[0] != expectedLocation && val[0] != expectedLocationBucketDNS { logError(testName, function, args, startTime, "", "Location in header response is incorrect", err) return } @@ -2588,6 +2583,10 @@ func testCopyObject() { return } + // Close all the get readers before proceeding with CopyObject operations. + r.Close() + readerCopy.Close() + // CopyObject again but with wrong conditions src = minio.NewSourceInfo(bucketName, objectName, nil) err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) @@ -2608,6 +2607,43 @@ func testCopyObject() { return } + // Perform the Copy which should update only metadata. + src = minio.NewSourceInfo(bucketName, objectName, nil) + dst, err = minio.NewDestinationInfo(bucketName, objectName, nil, map[string]string{ + "Copy": "should be same", + }) + args["dst"] = dst + args["src"] = src + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + + err = c.CopyObject(dst, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject shouldn't fail", err) + return + } + + oi, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + + stOpts := minio.StatObjectOptions{} + stOpts.SetMatchETag(oi.ETag) + objInfo, err = c.StatObject(bucketName, objectName, stOpts) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject ETag should match and not fail", err) + return + } + + if objInfo.Metadata.Get("x-amz-meta-copy") != "should be same" { + logError(testName, function, args, startTime, "", "CopyObject modified metadata should match", err) + return + } + // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) @@ -2620,23 +2656,19 @@ func testCopyObject() { successLogger(testName, function, args, startTime).Info() } -// TestEncryptionPutGet tests client side encryption -func testEncryptionPutGet() { +// Tests SSE-C get object ReaderSeeker interface methods. +func testEncryptedGetObjectReadSeekFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "PutEncryptedObject(bucketName, objectName, reader, cbcMaterials, metadata, progress)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "cbcMaterials": "", - "metadata": "", - } + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + // Seed random based on current time. rand.Seed(time.Now().Unix()) - // Instantiate new minio client object - c, err := minio.NewV4( + // Instantiate new minio client object. + c, err := minio.New( os.Getenv(serverEndpoint), os.Getenv(accessKey), os.Getenv(secretKey), @@ -2664,156 +2696,167 @@ func testEncryptionPutGet() { return } - // Generate a symmetric key - symKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) + defer func() { + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + }() + + // Generate 65MiB of data. + bufSize := dataFileMap["datafile-65-MB"] + var reader = getDataReader("datafile-65-MB") + defer reader.Close() - // Generate an assymmetric key from predefine public and private certificates - privateKey, err := hex.DecodeString( - "30820277020100300d06092a864886f70d0101010500048202613082025d" + - "0201000281810087b42ea73243a3576dc4c0b6fa245d339582dfdbddc20c" + - "bb8ab666385034d997210c54ba79275c51162a1221c3fb1a4c7c61131ca6" + - "5563b319d83474ef5e803fbfa7e52b889e1893b02586b724250de7ac6351" + - "cc0b7c638c980acec0a07020a78eed7eaa471eca4b92071394e061346c06" + - "15ccce2f465dee2080a89e43f29b5702030100010281801dd5770c3af8b3" + - "c85cd18cacad81a11bde1acfac3eac92b00866e142301fee565365aa9af4" + - "57baebf8bb7711054d071319a51dd6869aef3848ce477a0dc5f0dbc0c336" + - "5814b24c820491ae2bb3c707229a654427e03307fec683e6b27856688f08" + - "bdaa88054c5eeeb773793ff7543ee0fb0e2ad716856f2777f809ef7e6fa4" + - "41024100ca6b1edf89e8a8f93cce4b98c76c6990a09eb0d32ad9d3d04fbf" + - "0b026fa935c44f0a1c05dd96df192143b7bda8b110ec8ace28927181fd8c" + - "d2f17330b9b63535024100aba0260afb41489451baaeba423bee39bcbd1e" + - "f63dd44ee2d466d2453e683bf46d019a8baead3a2c7fca987988eb4d565e" + - "27d6be34605953f5034e4faeec9bdb0241009db2cb00b8be8c36710aff96" + - "6d77a6dec86419baca9d9e09a2b761ea69f7d82db2ae5b9aae4246599bb2" + - "d849684d5ab40e8802cfe4a2b358ad56f2b939561d2902404e0ead9ecafd" + - "bb33f22414fa13cbcc22a86bdf9c212ce1a01af894e3f76952f36d6c904c" + - "bd6a7e0de52550c9ddf31f1e8bfe5495f79e66a25fca5c20b3af5b870241" + - "0083456232aa58a8c45e5b110494599bda8dbe6a094683a0539ddd24e19d" + - "47684263bbe285ad953d725942d670b8f290d50c0bca3d1dc9688569f1d5" + - "9945cb5c7d") + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + buf, err := ioutil.ReadAll(reader) if err != nil { - logError(testName, function, args, startTime, "", "DecodeString for symmetric Key generation failed", err) + logError(testName, function, args, startTime, "", "ReadAll failed", err) return } - publicKey, err := hex.DecodeString("30819f300d06092a864886f70d010101050003818d003081890281810087" + - "b42ea73243a3576dc4c0b6fa245d339582dfdbddc20cbb8ab666385034d9" + - "97210c54ba79275c51162a1221c3fb1a4c7c61131ca65563b319d83474ef" + - "5e803fbfa7e52b889e1893b02586b724250de7ac6351cc0b7c638c980ace" + - "c0a07020a78eed7eaa471eca4b92071394e061346c0615ccce2f465dee20" + - "80a89e43f29b570203010001") + // Save the data + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + ContentType: "binary/octet-stream", + ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), + }) if err != nil { - logError(testName, function, args, startTime, "", "DecodeString for symmetric Key generation failed", err) + logError(testName, function, args, startTime, "", "PutObject failed", err) return } - // Generate an asymmetric key - asymKey, err := encrypt.NewAsymmetricKey(privateKey, publicKey) - if err != nil { - logError(testName, function, args, startTime, "", "NewAsymmetricKey for symmetric Key generation failed", err) + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) return } - testCases := []struct { - buf []byte - encKey encrypt.Key - }{ - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 0)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 15)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 16)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 17)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 31)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 32)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 33)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*2)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, - - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 0)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 16)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 32)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ + ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), + }) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return } + defer r.Close() - for i, testCase := range testCases { - // Generate a random object name - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat object failed", err) + return + } - // Secured object - cbcMaterials, err := encrypt.NewCBCSecureMaterials(testCase.encKey) - args["cbcMaterials"] = cbcMaterials + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + return + } - if err != nil { - logError(testName, function, args, startTime, "", "NewCBCSecureMaterials failed", err) + // This following function helps us to compare data from the reader after seek + // with the data from the original buffer + cmpData := func(r io.Reader, start, end int) { + if end-start == 0 { return } - - // Put encrypted data - _, err = c.PutEncryptedObject(bucketName, objectName, bytes.NewReader(testCase.buf), cbcMaterials) - if err != nil { - logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) - return + buffer := bytes.NewBuffer([]byte{}) + if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "CopyN failed", err) + return + } } - - // Read the data back - r, err := c.GetEncryptedObject(bucketName, objectName, cbcMaterials) - if err != nil { - logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) + if !bytes.Equal(buf[start:end], buffer.Bytes()) { + logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) return } - defer r.Close() + } - // Compare the sent object with the received one - recvBuffer := bytes.NewBuffer([]byte{}) - if _, err = io.Copy(recvBuffer, r); err != nil { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) + testCases := []struct { + offset int64 + whence int + pos int64 + err error + shouldCmp bool + start int + end int + }{ + // Start from offset 0, fetch data and compare + {0, 0, 0, nil, true, 0, 0}, + // Start from offset 2048, fetch data and compare + {2048, 0, 2048, nil, true, 2048, bufSize}, + // Start from offset larger than possible + {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, + // Move to offset 0 without comparing + {0, 0, 0, nil, false, 0, 0}, + // Move one step forward and compare + {1, 1, 1, nil, true, 1, bufSize}, + // Move larger than possible + {int64(bufSize), 1, 0, io.EOF, false, 0, 0}, + // Provide negative offset with CUR_SEEK + {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, + // Test with whence SEEK_END and with positive offset + {1024, 2, 0, io.EOF, false, 0, 0}, + // Test with whence SEEK_END and with negative offset + {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, + // Test with whence SEEK_END and with large negative offset + {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, + // Test with invalid whence + {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, + } + + for i, testCase := range testCases { + // Perform seek operation + n, err := r.Seek(testCase.offset, testCase.whence) + if err != nil && testCase.err == nil { + // We expected success. + logError(testName, function, args, startTime, "", + fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } - if recvBuffer.Len() != len(testCase.buf) { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) + if err == nil && testCase.err != nil { + // We expected failure, but got success. + logError(testName, function, args, startTime, "", + fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } - if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) + if err != nil && testCase.err != nil { + if err.Error() != testCase.err.Error() { + // We expect a specific error + logError(testName, function, args, startTime, "", + fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) + return + } + } + // Check the returned seek pos + if n != testCase.pos { + logError(testName, function, args, startTime, "", + fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err) return } - - successLogger(testName, function, args, startTime).Info() - - } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return + // Compare only if shouldCmp is activated + if testCase.shouldCmp { + cmpData(r, testCase.start, testCase.end) + } } successLogger(testName, function, args, startTime).Info() } -// TestEncryptionFPut tests client side encryption -func testEncryptionFPut() { +// Tests SSE-C get object ReaderAt interface methods. +func testEncryptedGetObjectReadAtFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, cbcMaterials)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "filePath": "", - "contentType": "", - "cbcMaterials": "", - } + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + // Seed random based on current time. rand.Seed(time.Now().Unix()) - // Instantiate new minio client object - c, err := minio.NewV4( + // Instantiate new minio client object. + c, err := minio.New( os.Getenv(serverEndpoint), os.Getenv(accessKey), os.Getenv(secretKey), @@ -2841,171 +2884,417 @@ func testEncryptionFPut() { return } - // Generate a symmetric key - symKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) + // Generate 65MiB of data. + bufSize := dataFileMap["datafile-65-MB"] + var reader = getDataReader("datafile-65-MB") + defer reader.Close() - // Generate an assymmetric key from predefine public and private certificates - privateKey, err := hex.DecodeString( - "30820277020100300d06092a864886f70d0101010500048202613082025d" + - "0201000281810087b42ea73243a3576dc4c0b6fa245d339582dfdbddc20c" + - "bb8ab666385034d997210c54ba79275c51162a1221c3fb1a4c7c61131ca6" + - "5563b319d83474ef5e803fbfa7e52b889e1893b02586b724250de7ac6351" + - "cc0b7c638c980acec0a07020a78eed7eaa471eca4b92071394e061346c06" + - "15ccce2f465dee2080a89e43f29b5702030100010281801dd5770c3af8b3" + - "c85cd18cacad81a11bde1acfac3eac92b00866e142301fee565365aa9af4" + - "57baebf8bb7711054d071319a51dd6869aef3848ce477a0dc5f0dbc0c336" + - "5814b24c820491ae2bb3c707229a654427e03307fec683e6b27856688f08" + - "bdaa88054c5eeeb773793ff7543ee0fb0e2ad716856f2777f809ef7e6fa4" + - "41024100ca6b1edf89e8a8f93cce4b98c76c6990a09eb0d32ad9d3d04fbf" + - "0b026fa935c44f0a1c05dd96df192143b7bda8b110ec8ace28927181fd8c" + - "d2f17330b9b63535024100aba0260afb41489451baaeba423bee39bcbd1e" + - "f63dd44ee2d466d2453e683bf46d019a8baead3a2c7fca987988eb4d565e" + - "27d6be34605953f5034e4faeec9bdb0241009db2cb00b8be8c36710aff96" + - "6d77a6dec86419baca9d9e09a2b761ea69f7d82db2ae5b9aae4246599bb2" + - "d849684d5ab40e8802cfe4a2b358ad56f2b939561d2902404e0ead9ecafd" + - "bb33f22414fa13cbcc22a86bdf9c212ce1a01af894e3f76952f36d6c904c" + - "bd6a7e0de52550c9ddf31f1e8bfe5495f79e66a25fca5c20b3af5b870241" + - "0083456232aa58a8c45e5b110494599bda8dbe6a094683a0539ddd24e19d" + - "47684263bbe285ad953d725942d670b8f290d50c0bca3d1dc9688569f1d5" + - "9945cb5c7d") + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + buf, err := ioutil.ReadAll(reader) if err != nil { - logError(testName, function, args, startTime, "", "DecodeString for symmetric Key generation failed", err) + logError(testName, function, args, startTime, "", "ReadAll failed", err) return } - publicKey, err := hex.DecodeString("30819f300d06092a864886f70d010101050003818d003081890281810087" + - "b42ea73243a3576dc4c0b6fa245d339582dfdbddc20cbb8ab666385034d9" + - "97210c54ba79275c51162a1221c3fb1a4c7c61131ca65563b319d83474ef" + - "5e803fbfa7e52b889e1893b02586b724250de7ac6351cc0b7c638c980ace" + - "c0a07020a78eed7eaa471eca4b92071394e061346c0615ccce2f465dee20" + - "80a89e43f29b570203010001") + // Save the data + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + ContentType: "binary/octet-stream", + ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), + }) if err != nil { - logError(testName, function, args, startTime, "", "DecodeString for symmetric Key generation failed", err) + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) return } - // Generate an asymmetric key - asymKey, err := encrypt.NewAsymmetricKey(privateKey, publicKey) + // read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ + ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), + }) if err != nil { - logError(testName, function, args, startTime, "", "NewAsymmetricKey for symmetric Key generation failed", err) + logError(testName, function, args, startTime, "", "PutObject failed", err) return } + defer r.Close() - // Object custom metadata - customContentType := "custom/contenttype" - args["metadata"] = customContentType + offset := int64(2048) - testCases := []struct { - buf []byte - encKey encrypt.Key - }{ - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 0)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 15)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 16)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 17)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 31)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 32)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 33)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*2)}, - {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, - - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 0)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 16)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 32)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024)}, - {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, - } + // read directly + buf1 := make([]byte, 512) + buf2 := make([]byte, 512) + buf3 := make([]byte, 512) + buf4 := make([]byte, 512) - for i, testCase := range testCases { - // Generate a random object name - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName + // Test readAt before stat is called such that objectInfo doesn't change. + m, err := r.ReadAt(buf1, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf1) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) + return + } + if !bytes.Equal(buf1, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + offset += 512 - // Secured object - cbcMaterials, err := encrypt.NewCBCSecureMaterials(testCase.encKey) - args["cbcMaterials"] = cbcMaterials + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } - if err != nil { - logError(testName, function, args, startTime, "", "NewCBCSecureMaterials failed", err) - return - } - // Generate a random file name. - fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - file, err := os.Create(fileName) - if err != nil { - logError(testName, function, args, startTime, "", "file create failed", err) - return - } - _, err = file.Write(testCase.buf) - if err != nil { - logError(testName, function, args, startTime, "", "file write failed", err) - return - } - file.Close() - // Put encrypted data - if _, err = c.FPutEncryptedObject(bucketName, objectName, fileName, cbcMaterials); err != nil { - logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) - return - } + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + return + } - // Read the data back - r, err := c.GetEncryptedObject(bucketName, objectName, cbcMaterials) - if err != nil { - logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) - return - } - defer r.Close() + m, err = r.ReadAt(buf2, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf2) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) + return + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + offset += 512 + m, err = r.ReadAt(buf3, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf3) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) + return + } + if !bytes.Equal(buf3, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + offset += 512 + m, err = r.ReadAt(buf4, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf4) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) + return + } + if !bytes.Equal(buf4, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } - // Compare the sent object with the received one - recvBuffer := bytes.NewBuffer([]byte{}) - if _, err = io.Copy(recvBuffer, r); err != nil { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) - return - } - if recvBuffer.Len() != len(testCase.buf) { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) - return - } - if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { - logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) + buf5 := make([]byte, n) + // Read the whole object. + m, err = r.ReadAt(buf5, 0) + if err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "ReadAt failed", err) return } + } + if m != len(buf5) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) + return + } + if !bytes.Equal(buf, buf5) { + logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) + return + } - if err = os.Remove(fileName); err != nil { - logError(testName, function, args, startTime, "", "File remove failed", err) + buf6 := make([]byte, n+1) + // Read the whole object and beyond. + _, err = r.ReadAt(buf6, 0) + if err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } - // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) return } - successLogger(testName, function, args, startTime).Info() } -func testBucketNotification() { +// TestEncryptionPutGet tests client side encryption +func testEncryptionPutGet() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "SetBucketNotification(bucketName)" + function := "PutEncryptedObject(bucketName, objectName, reader, sse)" args := map[string]interface{}{ "bucketName": "", + "objectName": "", + "sse": "", } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) - if os.Getenv("NOTIFY_BUCKET") == "" || - os.Getenv("NOTIFY_SERVICE") == "" || - os.Getenv("NOTIFY_REGION") == "" || - os.Getenv("NOTIFY_ACCOUNTID") == "" || - os.Getenv("NOTIFY_RESOURCE") == "" { - ignoredLog(testName, function, args, startTime, "Skipped notification test as it is not configured").Info() - return + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + testCases := []struct { + buf []byte + }{ + {buf: bytes.Repeat([]byte("F"), 1)}, + {buf: bytes.Repeat([]byte("F"), 15)}, + {buf: bytes.Repeat([]byte("F"), 16)}, + {buf: bytes.Repeat([]byte("F"), 17)}, + {buf: bytes.Repeat([]byte("F"), 31)}, + {buf: bytes.Repeat([]byte("F"), 32)}, + {buf: bytes.Repeat([]byte("F"), 33)}, + {buf: bytes.Repeat([]byte("F"), 1024)}, + {buf: bytes.Repeat([]byte("F"), 1024*2)}, + {buf: bytes.Repeat([]byte("F"), 1024*1024)}, + } + + const password = "correct horse battery staple" // https://xkcd.com/936/ + + for i, testCase := range testCases { + // Generate a random object name + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Secured object + sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) + args["sse"] = sse + + // Put encrypted data + _, err = c.PutObject(bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) + if err != nil { + logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) + if err != nil { + logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) + return + } + defer r.Close() + + // Compare the sent object with the received one + recvBuffer := bytes.NewBuffer([]byte{}) + if _, err = io.Copy(recvBuffer, r); err != nil { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) + return + } + if recvBuffer.Len() != len(testCase.buf) { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) + return + } + if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) + return + } + + successLogger(testName, function, args, startTime).Info() + + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// TestEncryptionFPut tests client side encryption +func testEncryptionFPut() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "filePath": "", + "contentType": "", + "sse": "", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Object custom metadata + customContentType := "custom/contenttype" + args["metadata"] = customContentType + + testCases := []struct { + buf []byte + }{ + {buf: bytes.Repeat([]byte("F"), 0)}, + {buf: bytes.Repeat([]byte("F"), 1)}, + {buf: bytes.Repeat([]byte("F"), 15)}, + {buf: bytes.Repeat([]byte("F"), 16)}, + {buf: bytes.Repeat([]byte("F"), 17)}, + {buf: bytes.Repeat([]byte("F"), 31)}, + {buf: bytes.Repeat([]byte("F"), 32)}, + {buf: bytes.Repeat([]byte("F"), 33)}, + {buf: bytes.Repeat([]byte("F"), 1024)}, + {buf: bytes.Repeat([]byte("F"), 1024*2)}, + {buf: bytes.Repeat([]byte("F"), 1024*1024)}, + } + + const password = "correct horse battery staple" // https://xkcd.com/936/ + for i, testCase := range testCases { + // Generate a random object name + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Secured object + sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) + args["sse"] = sse + + // Generate a random file name. + fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + file, err := os.Create(fileName) + if err != nil { + logError(testName, function, args, startTime, "", "file create failed", err) + return + } + _, err = file.Write(testCase.buf) + if err != nil { + logError(testName, function, args, startTime, "", "file write failed", err) + return + } + file.Close() + // Put encrypted data + if _, err = c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { + logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) + if err != nil { + logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) + return + } + defer r.Close() + + // Compare the sent object with the received one + recvBuffer := bytes.NewBuffer([]byte{}) + if _, err = io.Copy(recvBuffer, r); err != nil { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) + return + } + if recvBuffer.Len() != len(testCase.buf) { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) + return + } + if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) + return + } + + if err = os.Remove(fileName); err != nil { + logError(testName, function, args, startTime, "", "File remove failed", err) + return + } + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testBucketNotification() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "SetBucketNotification(bucketName)" + args := map[string]interface{}{ + "bucketName": "", + } + + if os.Getenv("NOTIFY_BUCKET") == "" || + os.Getenv("NOTIFY_SERVICE") == "" || + os.Getenv("NOTIFY_REGION") == "" || + os.Getenv("NOTIFY_ACCOUNTID") == "" || + os.Getenv("NOTIFY_RESOURCE") == "" { + ignoredLog(testName, function, args, startTime, "Skipped notification test as it is not configured").Info() + return } // Seed random based on current time. @@ -3101,7 +3390,7 @@ func testFunctional() { startTime := time.Now() testName := getFuncName() function := "testFunctional()" - function_all := "" + functionAll := "" args := map[string]interface{}{} // Seed random based on current time. @@ -3129,7 +3418,7 @@ func testFunctional() { // Make a new bucket. function = "MakeBucket(bucketName, region)" - function_all = "MakeBucket(bucketName, region)" + functionAll = "MakeBucket(bucketName, region)" args["bucketName"] = bucketName err = c.MakeBucket(bucketName, "us-east-1") @@ -3158,7 +3447,7 @@ func testFunctional() { // Verify if bucket exits and you have access. var exists bool function = "BucketExists(bucketName)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } @@ -3174,120 +3463,107 @@ func testFunctional() { } // Asserting the default bucket policy. - function = "GetBucketPolicy(bucketName, objectPrefix)" - function_all += ", " + function + function = "GetBucketPolicy(bucketName)" + functionAll += ", " + function args = map[string]interface{}{ - "bucketName": bucketName, - "objectPrefix": "", + "bucketName": bucketName, } - policyAccess, err := c.GetBucketPolicy(bucketName, "") - + nilPolicy, err := c.GetBucketPolicy(bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return } - if policyAccess != "none" { - logError(testName, function, args, startTime, "", "policy should be set to none", err) + if nilPolicy != "" { + logError(testName, function, args, startTime, "", "policy should be set to nil", err) return } // Set the bucket policy to 'public readonly'. - function = "SetBucketPolicy(bucketName, objectPrefix, bucketPolicy)" - function_all += ", " + function + function = "SetBucketPolicy(bucketName, readOnlyPolicy)" + functionAll += ", " + function + + readOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` args = map[string]interface{}{ "bucketName": bucketName, - "objectPrefix": "", - "bucketPolicy": policy.BucketPolicyReadOnly, + "bucketPolicy": readOnlyPolicy, } - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadOnly) + err = c.SetBucketPolicy(bucketName, readOnlyPolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // should return policy `readonly`. - function = "GetBucketPolicy(bucketName, objectPrefix)" - function_all += ", " + function + function = "GetBucketPolicy(bucketName)" + functionAll += ", " + function args = map[string]interface{}{ - "bucketName": bucketName, - "objectPrefix": "", + "bucketName": bucketName, } - policyAccess, err = c.GetBucketPolicy(bucketName, "") - + _, err = c.GetBucketPolicy(bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return } - if policyAccess != "readonly" { - logError(testName, function, args, startTime, "", "policy should be set to readonly", err) - return - } // Make the bucket 'public writeonly'. - function = "SetBucketPolicy(bucketName, objectPrefix, bucketPolicy)" - function_all += ", " + function + function = "SetBucketPolicy(bucketName, writeOnlyPolicy)" + functionAll += ", " + function + + writeOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` args = map[string]interface{}{ "bucketName": bucketName, - "objectPrefix": "", - "bucketPolicy": policy.BucketPolicyWriteOnly, + "bucketPolicy": writeOnlyPolicy, } - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyWriteOnly) + err = c.SetBucketPolicy(bucketName, writeOnlyPolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // should return policy `writeonly`. - function = "GetBucketPolicy(bucketName, objectPrefix)" - function_all += ", " + function + function = "GetBucketPolicy(bucketName)" + functionAll += ", " + function args = map[string]interface{}{ - "bucketName": bucketName, - "objectPrefix": "", + "bucketName": bucketName, } - policyAccess, err = c.GetBucketPolicy(bucketName, "") + _, err = c.GetBucketPolicy(bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return } - if policyAccess != "writeonly" { - logError(testName, function, args, startTime, "", "policy should be set to writeonly", err) - return - } + // Make the bucket 'public read/write'. - function = "SetBucketPolicy(bucketName, objectPrefix, bucketPolicy)" - function_all += ", " + function + function = "SetBucketPolicy(bucketName, readWritePolicy)" + functionAll += ", " + function + + readWritePolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` + args = map[string]interface{}{ "bucketName": bucketName, - "objectPrefix": "", - "bucketPolicy": policy.BucketPolicyReadWrite, + "bucketPolicy": readWritePolicy, } - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadWrite) + err = c.SetBucketPolicy(bucketName, readWritePolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // should return policy `readwrite`. - function = "GetBucketPolicy(bucketName, objectPrefix)" - function_all += ", " + function + function = "GetBucketPolicy(bucketName)" + functionAll += ", " + function args = map[string]interface{}{ - "bucketName": bucketName, - "objectPrefix": "", + "bucketName": bucketName, } - policyAccess, err = c.GetBucketPolicy(bucketName, "") - + _, err = c.GetBucketPolicy(bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return } - if policyAccess != "readwrite" { - logError(testName, function, args, startTime, "", "policy should be set to readwrite", err) - return - } + // List all buckets. function = "ListBuckets()" - function_all += ", " + function + functionAll += ", " + function args = nil buckets, err := c.ListBuckets() @@ -3320,7 +3596,7 @@ func testFunctional() { buf := bytes.Repeat([]byte("f"), 1<<19) function = "PutObject(bucketName, objectName, reader, contentType)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -3363,7 +3639,7 @@ func testFunctional() { isRecursive := true // Recursive is true. function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -3384,7 +3660,7 @@ func testFunctional() { objFound = false isRecursive = true // Recursive is true. function = "ListObjectsV2(bucketName, objectName, isRecursive, doneCh)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -3405,7 +3681,7 @@ func testFunctional() { incompObjNotFound := true function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -3424,7 +3700,7 @@ func testFunctional() { } function = "GetObject(bucketName, objectName)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -3446,9 +3722,10 @@ func testFunctional() { logError(testName, function, args, startTime, "", "GetObject bytes mismatch", err) return } + newReader.Close() function = "FGetObject(bucketName, objectName, fileName)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -3462,7 +3739,7 @@ func testFunctional() { } function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": "", @@ -3475,7 +3752,7 @@ func testFunctional() { // Generate presigned HEAD object url. function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -3504,7 +3781,7 @@ func testFunctional() { resp.Body.Close() function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": "", @@ -3518,7 +3795,7 @@ func testFunctional() { // Generate presigned GET object url. function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -3592,7 +3869,7 @@ func testFunctional() { } function = "PresignedPutObject(bucketName, objectName, expires)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": "", @@ -3605,7 +3882,7 @@ func testFunctional() { } function = "PresignedPutObject(bucketName, objectName, expires)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-presigned", @@ -3614,134 +3891,428 @@ func testFunctional() { presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) if err != nil { - logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) + logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) + return + } + + buf = bytes.Repeat([]byte("g"), 1<<19) + + req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) + if err != nil { + logError(testName, function, args, startTime, "", "Couldn't make HTTP request with PresignedPutObject URL", err) + return + } + httpClient := &http.Client{ + // Setting a sensible time out of 30secs to wait for response + // headers. Request is pro-actively cancelled after 30secs + // with no response. + Timeout: 30 * time.Second, + Transport: http.DefaultTransport, + } + resp, err = httpClient.Do(req) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) + return + } + + newReader, err = c.GetObject(bucketName, objectName+"-presigned", minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject after PresignedPutObject failed", err) + return + } + + newReadBytes, err = ioutil.ReadAll(newReader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll after GetObject failed", err) + return + } + + if !bytes.Equal(newReadBytes, buf) { + logError(testName, function, args, startTime, "", "Bytes mismatch", err) + return + } + + function = "RemoveObject(bucketName, objectName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + } + err = c.RemoveObject(bucketName, objectName) + + if err != nil { + logError(testName, function, args, startTime, "", "RemoveObject failed", err) + return + } + args["objectName"] = objectName + "-f" + err = c.RemoveObject(bucketName, objectName+"-f") + + if err != nil { + logError(testName, function, args, startTime, "", "RemoveObject failed", err) + return + } + + args["objectName"] = objectName + "-nolength" + err = c.RemoveObject(bucketName, objectName+"-nolength") + + if err != nil { + logError(testName, function, args, startTime, "", "RemoveObject failed", err) + return + } + + args["objectName"] = objectName + "-presigned" + err = c.RemoveObject(bucketName, objectName+"-presigned") + + if err != nil { + logError(testName, function, args, startTime, "", "RemoveObject failed", err) + return + } + + function = "RemoveBucket(bucketName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + } + err = c.RemoveBucket(bucketName) + + if err != nil { + logError(testName, function, args, startTime, "", "RemoveBucket failed", err) + return + } + err = c.RemoveBucket(bucketName) + if err == nil { + logError(testName, function, args, startTime, "", "RemoveBucket did not fail for invalid bucket name", err) + return + } + if err.Error() != "The specified bucket does not exist" { + logError(testName, function, args, startTime, "", "RemoveBucket failed", err) + return + } + + if err = os.Remove(fileName); err != nil { + logError(testName, function, args, startTime, "", "File Remove failed", err) + return + } + if err = os.Remove(fileName + "-f"); err != nil { + logError(testName, function, args, startTime, "", "File Remove failed", err) + return + } + successLogger(testName, functionAll, args, startTime).Info() +} + +// Test for validating GetObject Reader* methods functioning when the +// object is modified in the object store. +func testGetObjectModified() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + defer c.RemoveBucket(bucketName) + + // Upload an object. + objectName := "myobject" + args["objectName"] = objectName + content := "helloworld" + _, err = c.PutObject(bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"}) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) + return + } + + defer c.RemoveObject(bucketName, objectName) + + reader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to GetObject "+objectName+", from bucket "+bucketName, err) + return + } + defer reader.Close() + + // Read a few bytes of the object. + b := make([]byte, 5) + n, err := reader.ReadAt(b, 0) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to read object "+objectName+", from bucket "+bucketName+" at an offset", err) + return + } + + // Upload different contents to the same object while object is being read. + newContent := "goodbyeworld" + _, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"}) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) + return + } + + // Confirm that a Stat() call in between doesn't change the Object's cached etag. + _, err = reader.Stat() + expectedError := "At least one of the pre-conditions you specified did not hold" + if err.Error() != expectedError { + logError(testName, function, args, startTime, "", "Expected Stat to fail with error "+expectedError+", but received "+err.Error(), err) + return + } + + // Read again only to find object contents have been modified since last read. + _, err = reader.ReadAt(b, int64(n)) + if err.Error() != expectedError { + logError(testName, function, args, startTime, "", "Expected ReadAt to fail with error "+expectedError+", but received "+err.Error(), err) + return + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test validates putObject to upload a file seeked at a given offset. +func testPutObjectUploadSeekedObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, fileToUpload, contentType)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "fileToUpload": "", + "contentType": "binary/octet-stream", + } + + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + defer c.RemoveBucket(bucketName) + + var tempfile *os.File + + if fileName := getMintDataDirFilePath("datafile-100-kB"); fileName != "" { + tempfile, err = os.Open(fileName) + if err != nil { + logError(testName, function, args, startTime, "", "File open failed", err) + return + } + args["fileToUpload"] = fileName + } else { + tempfile, err = ioutil.TempFile("", "minio-go-upload-test-") + if err != nil { + logError(testName, function, args, startTime, "", "TempFile create failed", err) + return + } + args["fileToUpload"] = tempfile.Name() + + // Generate 100kB data + if _, err = io.Copy(tempfile, getDataReader("datafile-100-kB")); err != nil { + logError(testName, function, args, startTime, "", "File copy failed", err) + return + } + + defer os.Remove(tempfile.Name()) + + // Seek back to the beginning of the file. + tempfile.Seek(0, 0) + } + var length = 100 * humanize.KiByte + objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) + args["objectName"] = objectName + + offset := length / 2 + if _, err = tempfile.Seek(int64(offset), 0); err != nil { + logError(testName, function, args, startTime, "", "TempFile seek failed", err) + return + } + + n, err := c.PutObject(bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + if n != int64(length-offset) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid length returned, expected %d got %d", int64(length-offset), n), err) + return + } + tempfile.Close() + + obj, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) return } + defer obj.Close() - buf = bytes.Repeat([]byte("g"), 1<<19) - - req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) + n, err = obj.Seek(int64(offset), 0) if err != nil { - logError(testName, function, args, startTime, "", "Couldn't make HTTP request with PresignedPutObject URL", err) + logError(testName, function, args, startTime, "", "Seek failed", err) return } - httpClient := &http.Client{ - // Setting a sensible time out of 30secs to wait for response - // headers. Request is pro-actively cancelled after 30secs - // with no response. - Timeout: 30 * time.Second, - Transport: http.DefaultTransport, - } - resp, err = httpClient.Do(req) - if err != nil { - logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) + if n != int64(offset) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(offset), n), err) return } - newReader, err = c.GetObject(bucketName, objectName+"-presigned", minio.GetObjectOptions{}) + n, err = c.PutObject(bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { - logError(testName, function, args, startTime, "", "GetObject after PresignedPutObject failed", err) + logError(testName, function, args, startTime, "", "PutObject failed", err) return } - - newReadBytes, err = ioutil.ReadAll(newReader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll after GetObject failed", err) + if n != int64(length-offset) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(length-offset), n), err) return } - if !bytes.Equal(newReadBytes, buf) { - logError(testName, function, args, startTime, "", "Bytes mismatch", err) + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) return } - function = "RemoveObject(bucketName, objectName)" - function_all += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - "objectName": objectName, - } - err = c.RemoveObject(bucketName, objectName) + successLogger(testName, function, args, startTime).Info() +} - if err != nil { - logError(testName, function, args, startTime, "", "RemoveObject failed", err) - return +// Tests bucket re-create errors. +func testMakeBucketErrorV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "MakeBucket(bucketName, region)" + args := map[string]interface{}{ + "bucketName": "", + "region": "eu-west-1", } - args["objectName"] = objectName + "-f" - err = c.RemoveObject(bucketName, objectName+"-f") - if err != nil { - logError(testName, function, args, startTime, "", "RemoveObject failed", err) + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() return } - args["objectName"] = objectName + "-nolength" - err = c.RemoveObject(bucketName, objectName+"-nolength") + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) if err != nil { - logError(testName, function, args, startTime, "", "RemoveObject failed", err) + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) return } - args["objectName"] = objectName + "-presigned" - err = c.RemoveObject(bucketName, objectName+"-presigned") + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) - if err != nil { - logError(testName, function, args, startTime, "", "RemoveObject failed", err) - return - } + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - function = "RemoveBucket(bucketName)" - function_all += ", " + function - args = map[string]interface{}{ - "bucketName": bucketName, - } - err = c.RemoveBucket(bucketName) + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + region := "eu-west-1" + args["bucketName"] = bucketName + args["region"] = region - if err != nil { - logError(testName, function, args, startTime, "", "RemoveBucket failed", err) - return - } - err = c.RemoveBucket(bucketName) - if err == nil { - logError(testName, function, args, startTime, "", "RemoveBucket did not fail for invalid bucket name", err) + // Make a new bucket in 'eu-west-1'. + if err = c.MakeBucket(bucketName, region); err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } - if err.Error() != "The specified bucket does not exist" { - logError(testName, function, args, startTime, "", "RemoveBucket failed", err) + if err = c.MakeBucket(bucketName, region); err == nil { + logError(testName, function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err) return } - - if err = os.Remove(fileName); err != nil { - logError(testName, function, args, startTime, "", "File Remove failed", err) - return + // Verify valid error response from server. + if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && + minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { + logError(testName, function, args, startTime, "", "Invalid error returned by server", err) } - if err = os.Remove(fileName + "-f"); err != nil { - logError(testName, function, args, startTime, "", "File Remove failed", err) + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) return } - successLogger(testName, function_all, args, startTime).Info() + + successLogger(testName, function, args, startTime).Info() } -// Test for validating GetObject Reader* methods functioning when the -// object is modified in the object store. -func testGetObjectModified() { +// Test get object reader to not throw error on being closed twice. +func testGetObjectClosedTwiceV2() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "GetObject(bucketName, objectName)" - args := map[string]interface{}{} + function := "MakeBucket(bucketName, region)" + args := map[string]interface{}{ + "bucketName": "", + "region": "eu-west-1", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) // Instantiate new minio client object. - c, err := minio.NewV4( + c, err := minio.NewV2( os.Getenv(serverEndpoint), os.Getenv(accessKey), os.Getenv(secretKey), mustParseBool(os.Getenv(enableHTTPS)), ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) return } @@ -3751,66 +4322,63 @@ func testGetObjectModified() { // Set user agent. c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - // Make a new bucket. + // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName + // Make a new bucket. err = c.MakeBucket(bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } - defer c.RemoveBucket(bucketName) - // Upload an object. - objectName := "myobject" + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - content := "helloworld" - _, err = c.PutObject(bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"}) + + n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { - logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) + logError(testName, function, args, startTime, "", "PutObject failed", err) return } - defer c.RemoveObject(bucketName, objectName) - - reader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "Failed to GetObject "+objectName+", from bucket "+bucketName, err) + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) return } - defer reader.Close() - // Read a few bytes of the object. - b := make([]byte, 5) - n, err := reader.ReadAt(b, 0) + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "Failed to read object "+objectName+", from bucket "+bucketName+" at an offset", err) + logError(testName, function, args, startTime, "", "GetObject failed", err) return } - // Upload different contents to the same object while object is being read. - newContent := "goodbyeworld" - _, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"}) + st, err := r.Stat() if err != nil { - logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) + logError(testName, function, args, startTime, "", "Stat failed", err) return } - // Confirm that a Stat() call in between doesn't change the Object's cached etag. - _, err = reader.Stat() - expectedError := "At least one of the pre-conditions you specified did not hold" - if err.Error() != expectedError { - logError(testName, function, args, startTime, "", "Expected Stat to fail with error "+expectedError+", but received "+err.Error(), err) + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) return } - - // Read again only to find object contents have been modified since last read. - _, err = reader.ReadAt(b, int64(n)) - if err.Error() != expectedError { - logError(testName, function, args, startTime, "", "Expected ReadAt to fail with error "+expectedError+", but received "+err.Error(), err) + if err := r.Close(); err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + if err := r.Close(); err == nil { + logError(testName, function, args, startTime, "", "Object is already closed, should return error", err) return } + // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) @@ -3820,120 +4388,157 @@ func testGetObjectModified() { successLogger(testName, function, args, startTime).Info() } -// Test validates putObject to upload a file seeked at a given offset. -func testPutObjectUploadSeekedObject() { +// Tests FPutObject hidden contentType setting +func testFPutObjectV2() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "PutObject(bucketName, objectName, fileToUpload, contentType)" + function := "FPutObject(bucketName, objectName, fileName, opts)" args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "fileToUpload": "", - "contentType": "binary/octet-stream", + "bucketName": "", + "objectName": "", + "fileName": "", + "opts": "", } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + // Instantiate new minio client object. - c, err := minio.NewV4( + c, err := minio.NewV2( os.Getenv(serverEndpoint), os.Getenv(accessKey), os.Getenv(secretKey), mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Make a temp file with 11*1024*1024 bytes of data. + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + if err != nil { + logError(testName, function, args, startTime, "", "TempFile creation failed", err) + return + } + + r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024)) + n, err := io.CopyN(file, r, 11*1024*1024) + if err != nil { + logError(testName, function, args, startTime, "", "Copy failed", err) + return + } + if n != int64(11*1024*1024) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) return } - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Make a new bucket. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - err = c.MakeBucket(bucketName, "us-east-1") + // Close the file pro-actively for windows. + err = file.Close() if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) + logError(testName, function, args, startTime, "", "File close failed", err) return } - defer c.RemoveBucket(bucketName) - var tempfile *os.File - - if fileName := getMintDataDirFilePath("datafile-100-kB"); fileName != "" { - tempfile, err = os.Open(fileName) - if err != nil { - logError(testName, function, args, startTime, "", "File open failed", err) - return - } - args["fileToUpload"] = fileName - } else { - tempfile, err = ioutil.TempFile("", "minio-go-upload-test-") - if err != nil { - logError(testName, function, args, startTime, "", "TempFile create failed", err) - return - } - args["fileToUpload"] = tempfile.Name() + // Set base object name + objectName := bucketName + "FPutObject" + args["objectName"] = objectName + args["fileName"] = file.Name() - // Generate 100kB data - if _, err = io.Copy(tempfile, getDataReader("datafile-100-kB")); err != nil { - logError(testName, function, args, startTime, "", "File copy failed", err) - return - } + // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) + n, err = c.FPutObject(bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "FPutObject failed", err) + return + } + if n != int64(11*1024*1024) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) + return + } - defer os.Remove(tempfile.Name()) + // Perform FPutObject with no contentType provided (Expecting application/octet-stream) + args["objectName"] = objectName + "-Octet" + args["contentType"] = "" - // Seek back to the beginning of the file. - tempfile.Seek(0, 0) + n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "FPutObject failed", err) + return + } + if n != int64(11*1024*1024) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) + return } - var length = 100 * humanize.KiByte - objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) - args["objectName"] = objectName - offset := length / 2 - if _, err = tempfile.Seek(int64(offset), 0); err != nil { - logError(testName, function, args, startTime, "", "TempFile seek failed", err) + // Add extension to temp file name + fileName := file.Name() + err = os.Rename(file.Name(), fileName+".gtar") + if err != nil { + logError(testName, function, args, startTime, "", "Rename failed", err) return } - n, err := c.PutObject(bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + // Perform FPutObject with no contentType provided (Expecting application/x-gtar) + args["objectName"] = objectName + "-Octet" + args["contentType"] = "" + args["fileName"] = fileName + ".gtar" + + n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) + logError(testName, function, args, startTime, "", "FPutObject failed", err) return } - if n != int64(length-offset) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid length returned, expected %d got %d", int64(length-offset), n), err) + if n != int64(11*1024*1024) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) return } - tempfile.Close() - obj, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + // Check headers + rStandard, err := c.StatObject(bucketName, objectName+"-standard", minio.StatObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + if rStandard.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rStandard.ContentType, err) return } - n, err = obj.Seek(int64(offset), 0) + rOctet, err := c.StatObject(bucketName, objectName+"-Octet", minio.StatObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "Seek failed", err) + logError(testName, function, args, startTime, "", "StatObject failed", err) return } - if n != int64(offset) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(offset), n), err) + if rOctet.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rOctet.ContentType, err) return } - n, err = c.PutObject(bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + rGTar, err := c.StatObject(bucketName, objectName+"-GTar", minio.StatObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) + logError(testName, function, args, startTime, "", "StatObject failed", err) return } - if n != int64(length-offset) { - logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(length-offset), n), err) + if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err) return } @@ -3943,11 +4548,16 @@ func testPutObjectUploadSeekedObject() { return } + err = os.Remove(fileName + ".gtar") + if err != nil { + logError(testName, function, args, startTime, "", "File remove failed", err) + return + } successLogger(testName, function, args, startTime).Info() } -// Tests bucket re-create errors. -func testMakeBucketErrorV2() { +// Tests various bucket supported formats. +func testMakeBucketRegionsV2() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -3985,26 +4595,31 @@ func testMakeBucketErrorV2() { // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - region := "eu-west-1" args["bucketName"] = bucketName - args["region"] = region - // Make a new bucket in 'eu-west-1'. - if err = c.MakeBucket(bucketName, region); err != nil { + // Make a new bucket in 'eu-central-1'. + if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } - if err = c.MakeBucket(bucketName, region); err == nil { - logError(testName, function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err) + + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) return } - // Verify valid error response from server. - if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && - minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { - logError(testName, function, args, startTime, "", "Invalid error returned by server", err) + + // Make a new bucket with '.' in its name, in 'us-west-2'. This + // request is internally staged into a path style instead of + // virtual host style. + if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { + args["bucketName"] = bucketName + ".withperiod" + args["region"] = "us-west-2" + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return } + // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { + if err = cleanupBucket(bucketName+".withperiod", c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) return } @@ -4012,16 +4627,13 @@ func testMakeBucketErrorV2() { successLogger(testName, function, args, startTime).Info() } -// Test get object reader to not throw error on being closed twice. -func testGetObjectClosedTwiceV2() { +// Tests get object ReaderSeeker interface methods. +func testGetObjectReadSeekFunctionalV2() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "MakeBucket(bucketName, region)" - args := map[string]interface{}{ - "bucketName": "", - "region": "eu-west-1", - } + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} // Seed random based on current time. rand.Seed(time.Now().Unix()) @@ -4060,18 +4672,24 @@ func testGetObjectClosedTwiceV2() { var reader = getDataReader("datafile-33-kB") defer reader.Close() - // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + buf, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + // Save the data. + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) return } @@ -4081,6 +4699,7 @@ func testGetObjectClosedTwiceV2() { logError(testName, function, args, startTime, "", "GetObject failed", err) return } + defer r.Close() st, err := r.Stat() if err != nil { @@ -4089,101 +4708,80 @@ func testGetObjectClosedTwiceV2() { } if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) return } - if err := r.Close(); err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) + + offset := int64(2048) + n, err = r.Seek(offset, 0) + if err != nil { + logError(testName, function, args, startTime, "", "Seek failed", err) return } - if err := r.Close(); err == nil { - logError(testName, function, args, startTime, "", "Object is already closed, should return error", err) + if n != offset { + logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) return } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) + n, err = r.Seek(0, 1) + if err != nil { + logError(testName, function, args, startTime, "", "Seek failed", err) return } - - successLogger(testName, function, args, startTime).Info() -} - -// Tests removing partially uploaded objects. -func testRemovePartiallyUploadedV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "RemoveIncompleteUpload(bucketName, objectName)" - args := map[string]interface{}{} - - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + if n != offset { + logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) return } - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Enable tracing, write to stdout. - // c.TraceOn(os.Stderr) - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + _, err = r.Seek(offset, 2) + if err == nil { + logError(testName, function, args, startTime, "", "Seek on positive offset for whence '2' should error out", err) + return + } + n, err = r.Seek(-offset, 2) if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) + logError(testName, function, args, startTime, "", "Seek failed", err) + return + } + if n != st.Size-offset { + logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(st.Size-offset)+" got "+string(n), err) return } - r := bytes.NewReader(bytes.Repeat([]byte("a"), 128*1024)) - - reader, writer := io.Pipe() - go func() { - i := 0 - for i < 25 { - _, cerr := io.CopyN(writer, r, 128*1024) - if cerr != nil { - logError(testName, function, args, startTime, "", "Copy failed", cerr) - return - } - i++ - r.Seek(0, 0) - } - writer.CloseWithError(errors.New("proactively closed to be verified later")) - }() - - objectName := bucketName + "-resumable" - args["objectName"] = objectName + var buffer1 bytes.Buffer + if _, err = io.CopyN(&buffer1, r, st.Size); err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "Copy failed", err) + return + } + } + if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) { + logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) + return + } - _, err = c.PutObject(bucketName, objectName, reader, -1, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err == nil { - logError(testName, function, args, startTime, "", "PutObject should fail", err) + // Seek again and read again. + n, err = r.Seek(offset-1, 0) + if err != nil { + logError(testName, function, args, startTime, "", "Seek failed", err) return } - if err.Error() != "proactively closed to be verified later" { - logError(testName, function, args, startTime, "", "Unexpected error, expected : proactively closed to be verified later", err) + if n != (offset - 1) { + logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset-1)+" got "+string(n), err) return } - err = c.RemoveIncompleteUpload(bucketName, objectName) - if err != nil { - logError(testName, function, args, startTime, "", "RemoveIncompleteUpload failed", err) + + var buffer2 bytes.Buffer + if _, err = io.CopyN(&buffer2, r, st.Size); err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "Copy failed", err) + return + } + } + // Verify now lesser bytes. + if !bytes.Equal(buf[2047:], buffer2.Bytes()) { + logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) return } + // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) @@ -4193,18 +4791,13 @@ func testRemovePartiallyUploadedV2() { successLogger(testName, function, args, startTime).Info() } -// Tests FPutObject hidden contentType setting -func testFPutObjectV2() { +// Tests get object ReaderAt interface methods. +func testGetObjectReadAtFunctionalV2() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "FPutObject(bucketName, objectName, fileName, opts)" - args := map[string]interface{}{ - "bucketName": "", - "objectName": "", - "fileName": "", - "opts": "", - } + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} // Seed random based on current time. rand.Seed(time.Now().Unix()) @@ -4238,193 +4831,129 @@ func testFPutObjectV2() { return } - // Make a temp file with 11*1024*1024 bytes of data. - file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + buf, err := ioutil.ReadAll(reader) if err != nil { - logError(testName, function, args, startTime, "", "TempFile creation failed", err) + logError(testName, function, args, startTime, "", "ReadAll failed", err) return } - r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024)) - n, err := io.CopyN(file, r, 11*1024*1024) + // Save the data + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { - logError(testName, function, args, startTime, "", "Copy failed", err) - return - } - if n != int64(11*1024*1024) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) + logError(testName, function, args, startTime, "", "PutObject failed", err) return } - // Close the file pro-actively for windows. - err = file.Close() - if err != nil { - logError(testName, function, args, startTime, "", "File close failed", err) + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) return } - // Set base object name - objectName := bucketName + "FPutObject" - args["objectName"] = objectName - args["fileName"] = file.Name() - - // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) - n, err = c.FPutObject(bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "FPutObject failed", err) - return - } - if n != int64(11*1024*1024) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) + logError(testName, function, args, startTime, "", "GetObject failed", err) return } + defer r.Close() - // Perform FPutObject with no contentType provided (Expecting application/octet-stream) - args["objectName"] = objectName + "-Octet" - args["contentType"] = "" - - n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{}) + st, err := r.Stat() if err != nil { - logError(testName, function, args, startTime, "", "FPutObject failed", err) - return - } - if n != int64(11*1024*1024) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) + logError(testName, function, args, startTime, "", "Stat failed", err) return } - // Add extension to temp file name - fileName := file.Name() - err = os.Rename(file.Name(), fileName+".gtar") - if err != nil { - logError(testName, function, args, startTime, "", "Rename failed", err) + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) return } - // Perform FPutObject with no contentType provided (Expecting application/x-gtar) - args["objectName"] = objectName + "-Octet" - args["contentType"] = "" - args["fileName"] = fileName + ".gtar" + offset := int64(2048) - n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "FPutObject failed", err) - return - } - if n != int64(11*1024*1024) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) - return - } + // Read directly + buf2 := make([]byte, 512) + buf3 := make([]byte, 512) + buf4 := make([]byte, 512) - // Check headers - rStandard, err := c.StatObject(bucketName, objectName+"-standard", minio.StatObjectOptions{}) + m, err := r.ReadAt(buf2, offset) if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) - return - } - if rStandard.ContentType != "application/octet-stream" { - logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rStandard.ContentType, err) + logError(testName, function, args, startTime, "", "ReadAt failed", err) return } - - rOctet, err := c.StatObject(bucketName, objectName+"-Octet", minio.StatObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) + if m != len(buf2) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+" got "+string(m), err) return } - if rOctet.ContentType != "application/octet-stream" { - logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rOctet.ContentType, err) + if !bytes.Equal(buf2, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } - - rGTar, err := c.StatObject(bucketName, objectName+"-GTar", minio.StatObjectOptions{}) + offset += 512 + m, err = r.ReadAt(buf3, offset) if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) + logError(testName, function, args, startTime, "", "ReadAt failed", err) return } - if rGTar.ContentType != "application/x-gtar" { - logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err) + if m != len(buf3) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+" got "+string(m), err) return } - - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) + if !bytes.Equal(buf3, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } - - err = os.Remove(fileName + ".gtar") + offset += 512 + m, err = r.ReadAt(buf4, offset) if err != nil { - logError(testName, function, args, startTime, "", "File remove failed", err) + logError(testName, function, args, startTime, "", "ReadAt failed", err) return } - successLogger(testName, function, args, startTime).Info() -} - -// Tests various bucket supported formats. -func testMakeBucketRegionsV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "MakeBucket(bucketName, region)" - args := map[string]interface{}{ - "bucketName": "", - "region": "eu-west-1", + if m != len(buf4) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+" got "+string(m), err) + return } - - if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { - ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() + if !bytes.Equal(buf4, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + buf5 := make([]byte, n) + // Read the whole object. + m, err = r.ReadAt(buf5, 0) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return + if err != io.EOF { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - - // Make a new bucket in 'eu-central-1'. - if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) + if m != len(buf5) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+" got "+string(m), err) return } - - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) + if !bytes.Equal(buf, buf5) { + logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) return } - // Make a new bucket with '.' in its name, in 'us-west-2'. This - // request is internally staged into a path style instead of - // virtual host style. - if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { - args["bucketName"] = bucketName + ".withperiod" - args["region"] = "us-west-2" - logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return + buf6 := make([]byte, n+1) + // Read the whole object and beyond. + _, err = r.ReadAt(buf6, 0) + if err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } } - // Delete all objects and buckets - if err = cleanupBucket(bucketName+".withperiod", c); err != nil { + if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) return } @@ -4432,18 +4961,18 @@ func testMakeBucketRegionsV2() { successLogger(testName, function, args, startTime).Info() } -// Tests get object ReaderSeeker interface methods. -func testGetObjectReadSeekFunctionalV2() { +// Tests copy object +func testCopyObjectV2() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "GetObject(bucketName, objectName)" + function := "CopyObject(destination, source)" args := map[string]interface{}{} // Seed random based on current time. rand.Seed(time.Now().Unix()) - // Instantiate new minio client object. + // Instantiate new minio client object c, err := minio.NewV2( os.Getenv(serverEndpoint), os.Getenv(accessKey), @@ -4463,31 +4992,29 @@ func testGetObjectReadSeekFunctionalV2() { // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - // Make a new bucket. + // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } + // Make a new bucket in 'us-east-1' (destination bucket). + err = c.MakeBucket(bucketName+"-copy", "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] var reader = getDataReader("datafile-33-kB") defer reader.Close() + // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) - if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - - // Save the data. - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -4498,91 +5025,121 @@ func testGetObjectReadSeekFunctionalV2() { return } - // Read the data back r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } - - st, err := r.Stat() + // Check the various fields of source object against destination object. + objInfo, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } + r.Close() - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) + // Copy Source + src := minio.NewSourceInfo(bucketName, objectName, nil) + args["source"] = src + + // Set copy conditions. + + // All invalid conditions first. + err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) + if err == nil { + logError(testName, function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err) + return + } + err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) + if err == nil { + logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err) + return + } + err = src.SetMatchETagCond("") + if err == nil { + logError(testName, function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err) + return + } + err = src.SetMatchETagExceptCond("") + if err == nil { + logError(testName, function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err) return } - offset := int64(2048) - n, err = r.Seek(offset, 0) + err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) if err != nil { - logError(testName, function, args, startTime, "", "Seek failed", err) + logError(testName, function, args, startTime, "", "SetModifiedSinceCond failed", err) return } - if n != offset { - logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) + err = src.SetMatchETagCond(objInfo.ETag) + if err != nil { + logError(testName, function, args, startTime, "", "SetMatchETagCond failed", err) return } - n, err = r.Seek(0, 1) + + dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) + args["destination"] = dst if err != nil { - logError(testName, function, args, startTime, "", "Seek failed", err) + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } - if n != offset { - logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) + + // Perform the Copy + err = c.CopyObject(dst, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) return } - _, err = r.Seek(offset, 2) - if err == nil { - logError(testName, function, args, startTime, "", "Seek on positive offset for whence '2' should error out", err) + + // Source object + r, err = c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) return } - n, err = r.Seek(-offset, 2) + // Destination object + readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "Seek failed", err) + logError(testName, function, args, startTime, "", "GetObject failed", err) return } - if n != st.Size-offset { - logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(st.Size-offset)+" got "+string(n), err) + // Check the various fields of source object against destination object. + objInfo, err = r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) return } - - var buffer1 bytes.Buffer - if _, err = io.CopyN(&buffer1, r, st.Size); err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "Copy failed", err) - return - } + objInfoCopy, err := readerCopy.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return } - if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) { - logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) + if objInfo.Size != objInfoCopy.Size { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+" got "+string(objInfo.Size), err) return } - // Seek again and read again. - n, err = r.Seek(offset-1, 0) + // Close all the readers. + r.Close() + readerCopy.Close() + + // CopyObject again but with wrong conditions + src = minio.NewSourceInfo(bucketName, objectName, nil) + err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) if err != nil { - logError(testName, function, args, startTime, "", "Seek failed", err) + logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond failed", err) return } - if n != (offset - 1) { - logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset-1)+" got "+string(n), err) + err = src.SetMatchETagExceptCond(objInfo.ETag) + if err != nil { + logError(testName, function, args, startTime, "", "SetMatchETagExceptCond failed", err) return } - var buffer2 bytes.Buffer - if _, err = io.CopyN(&buffer2, r, st.Size); err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "Copy failed", err) - return - } - } - // Verify now lesser bytes. - if !bytes.Equal(buf[2047:], buffer2.Bytes()) { - logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) + // Perform the Copy which should fail + err = c.CopyObject(dst, src) + if err == nil { + logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) return } @@ -4591,190 +5148,192 @@ func testGetObjectReadSeekFunctionalV2() { logError(testName, function, args, startTime, "", "Cleanup failed", err) return } - + if err = cleanupBucket(bucketName+"-copy", c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } successLogger(testName, function, args, startTime).Info() } -// Tests get object ReaderAt interface methods. -func testGetObjectReadAtFunctionalV2() { +func testComposeObjectErrorCasesWrapper(c *minio.Client) { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "GetObject(bucketName, objectName)" + function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{} - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - - // Instantiate new minio client object. - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return - } - - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) - - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") - // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - args["bucketName"] = bucketName - // Make a new bucket. - err = c.MakeBucket(bucketName, "us-east-1") + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") + if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } - // Generate 33K of data. - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - args["objectName"] = objectName - - buf, err := ioutil.ReadAll(reader) + // Test that more than 10K source objects cannot be + // concatenated. + srcArr := [10001]minio.SourceInfo{} + srcSlice := srcArr[:] + dst, err := minio.NewDestinationInfo(bucketName, "object", nil, nil) if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } - // Save the data - n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + args["destination"] = dst + // Just explain about srcArr in args["sourceList"] + // to stop having 10,001 null headers logged + args["sourceList"] = "source array of 10,001 elements" + if err := c.ComposeObject(dst, srcSlice); err == nil { + logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err) + return + } else if err.Error() != "There must be as least one and up to 10000 source objects." { + logError(testName, function, args, startTime, "", "Got unexpected error", err) + return + } + + // Create a source with invalid offset spec and check that + // error is returned: + // 1. Create the source object. + const badSrcSize = 5 * 1024 * 1024 + buf := bytes.Repeat([]byte("1"), badSrcSize) + _, err = c.PutObject(bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } + // 2. Set invalid range spec on the object (going beyond + // object size) + badSrc := minio.NewSourceInfo(bucketName, "badObject", nil) + err = badSrc.SetRange(1, badSrcSize) + if err != nil { + logError(testName, function, args, startTime, "", "Setting NewSourceInfo failed", err) + return + } + // 3. ComposeObject call should fail. + if err := c.ComposeObject(dst, []minio.SourceInfo{badSrc}); err == nil { + logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err) + return + } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { + logError(testName, function, args, startTime, "", "Got invalid error", err) + return + } - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) return } - // Read the data back - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + successLogger(testName, function, args, startTime).Info() +} + +// Test expected error cases +func testComposeObjectErrorCasesV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "ComposeObject(destination, sourceList)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + testComposeObjectErrorCasesWrapper(c) +} + +func testComposeMultipleSources(c *minio.Client) { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "ComposeObject(destination, sourceList)" + args := map[string]interface{}{ + "destination": "", + "sourceList": "", + } + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) + logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } - st, err := r.Stat() + // Upload a small source object + const srcSize = 1024 * 1024 * 5 + buf := bytes.Repeat([]byte("1"), srcSize) + _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) + logError(testName, function, args, startTime, "", "PutObject failed", err) return } - if st.Size != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) - return + // We will append 10 copies of the object. + srcs := []minio.SourceInfo{} + for i := 0; i < 10; i++ { + srcs = append(srcs, minio.NewSourceInfo(bucketName, "srcObject", nil)) } - - offset := int64(2048) - - // Read directly - buf2 := make([]byte, 512) - buf3 := make([]byte, 512) - buf4 := make([]byte, 512) - - m, err := r.ReadAt(buf2, offset) + // make the last part very small + err = srcs[9].SetRange(0, 0) if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf2) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+" got "+string(m), err) - return - } - if !bytes.Equal(buf2, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + logError(testName, function, args, startTime, "", "SetRange failed", err) return } - offset += 512 - m, err = r.ReadAt(buf3, offset) + args["sourceList"] = srcs + + dst, err := minio.NewDestinationInfo(bucketName, "dstObject", nil, nil) + args["destination"] = dst + if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf3) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+" got "+string(m), err) - return - } - if !bytes.Equal(buf3, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } - offset += 512 - m, err = r.ReadAt(buf4, offset) + err = c.ComposeObject(dst, srcs) if err != nil { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - if m != len(buf4) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+" got "+string(m), err) - return - } - if !bytes.Equal(buf4, buf[offset:offset+512]) { - logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + logError(testName, function, args, startTime, "", "ComposeObject failed", err) return } - buf5 := make([]byte, n) - // Read the whole object. - m, err = r.ReadAt(buf5, 0) + objProps, err := c.StatObject(bucketName, "dstObject", minio.StatObjectOptions{}) if err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } - } - if m != len(buf5) { - logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+" got "+string(m), err) - return - } - if !bytes.Equal(buf, buf5) { - logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) + logError(testName, function, args, startTime, "", "StatObject failed", err) return } - buf6 := make([]byte, n+1) - // Read the whole object and beyond. - _, err = r.ReadAt(buf6, 0) - if err != nil { - if err != io.EOF { - logError(testName, function, args, startTime, "", "ReadAt failed", err) - return - } + if objProps.Size != 9*srcSize+1 { + logError(testName, function, args, startTime, "", "Size mismatched! Expected "+string(10000*srcSize)+" got "+string(objProps.Size), err) + return } // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) return } - successLogger(testName, function, args, startTime).Info() } -// Tests copy object -func testCopyObjectV2() { +// Test concatenating multiple objects objects +func testCompose10KSourcesV2() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "CopyObject(destination, source)" + function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{} - // Seed random based on current time. - rand.Seed(time.Now().Unix()) - // Instantiate new minio client object c, err := minio.NewV2( os.Getenv(serverEndpoint), @@ -4787,15 +5346,31 @@ func testCopyObjectV2() { return } - // Enable tracing, write to stderr. - // c.TraceOn(os.Stderr) + testComposeMultipleSources(c) +} - // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") +func testEncryptedEmptyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader, objectSize, opts)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + return + } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - + args["bucketName"] = bucketName // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(bucketName, "us-east-1") if err != nil { @@ -4803,220 +5378,206 @@ func testCopyObjectV2() { return } - // Make a new bucket in 'us-east-1' (destination bucket). - err = c.MakeBucket(bucketName+"-copy", "us-east-1") + sse := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"object")) + + // 1. create an sse-c encrypted object to copy by uploading + const srcSize = 0 + var buf []byte // Empty buffer + args["objectName"] = "object" + _, err = c.PutObject(bucketName, "object", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) + logError(testName, function, args, startTime, "", "PutObject call failed", err) return } - // Generate 33K of data. - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - // Save the data - objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") - n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + // 2. Test CopyObject for an empty object + dstInfo, err := minio.NewDestinationInfo(bucketName, "new-object", sse, nil) if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) + args["objectName"] = "new-object" + function = "NewDestinationInfo(bucketName, objectName, sse, userMetadata)" + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } - - if n != int64(bufSize) { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) + srcInfo := minio.NewSourceInfo(bucketName, "object", sse) + if err = c.CopyObject(dstInfo, srcInfo); err != nil { + function = "CopyObject(dstInfo, srcInfo)" + logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject failed", err) return } - r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + // 3. Test Key rotation + newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"new-object")) + dstInfo, err = minio.NewDestinationInfo(bucketName, "new-object", newSSE, nil) if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) + args["objectName"] = "new-object" + function = "NewDestinationInfo(bucketName, objectName, encryptSSEC, userMetadata)" + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } - // Check the various fields of source object against destination object. - objInfo, err := r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) + + srcInfo = minio.NewSourceInfo(bucketName, "new-object", sse) + if err = c.CopyObject(dstInfo, srcInfo); err != nil { + function = "CopyObject(dstInfo, srcInfo)" + logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject with key rotation failed", err) return } - // Copy Source - src := minio.NewSourceInfo(bucketName, objectName, nil) - args["source"] = src - - // Set copy conditions. - - // All invalid conditions first. - err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logError(testName, function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err) + // 4. Download the object. + reader, err := c.GetObject(bucketName, "new-object", minio.GetObjectOptions{ServerSideEncryption: newSSE}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) return } - err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) - if err == nil { - logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err) + defer reader.Close() + + decBytes, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, map[string]interface{}{}, startTime, "", "ReadAll failed", err) return } - err = src.SetMatchETagCond("") - if err == nil { - logError(testName, function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err) + if !bytes.Equal(decBytes, buf) { + logError(testName, function, map[string]interface{}{}, startTime, "", "Downloaded object doesn't match the empty encrypted object", err) return } - err = src.SetMatchETagExceptCond("") - if err == nil { - logError(testName, function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err) + // Delete all objects and buckets + delete(args, "objectName") + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) return } - err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + successLogger(testName, function, args, startTime).Info() +} + +func testEncryptedCopyObjectWrapper(c *minio.Client) { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") if err != nil { - logError(testName, function, args, startTime, "", "SetModifiedSinceCond failed", err) + logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } - err = src.SetMatchETagCond(objInfo.ETag) + + sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) + sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) + + // 1. create an sse-c encrypted object to copy by uploading + const srcSize = 1024 * 1024 + buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB + _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + ServerSideEncryption: sseSrc, + }) if err != nil { - logError(testName, function, args, startTime, "", "SetMatchETagCond failed", err) + logError(testName, function, args, startTime, "", "PutObject call failed", err) return } - dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) - args["destination"] = dst + // 2. copy object and change encryption key + src := minio.NewSourceInfo(bucketName, "srcObject", sseSrc) + args["source"] = src + dst, err := minio.NewDestinationInfo(bucketName, "dstObject", sseDst, nil) if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } + args["destination"] = dst - // Perform the Copy err = c.CopyObject(dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } - // Source object - r, err = c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) - if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return - } - // Destination object - readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) + // 3. get copied object and check if content is equal + coreClient := minio.Core{c} + reader, _, err := coreClient.GetObject(bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: sseDst}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } - // Check the various fields of source object against destination object. - objInfo, err = r.Stat() - if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - objInfoCopy, err := readerCopy.Stat() + + decBytes, err := ioutil.ReadAll(reader) if err != nil { - logError(testName, function, args, startTime, "", "Stat failed", err) - return - } - if objInfo.Size != objInfoCopy.Size { - logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+" got "+string(objInfo.Size), err) + logError(testName, function, args, startTime, "", "ReadAll failed", err) return } - - // CopyObject again but with wrong conditions - src = minio.NewSourceInfo(bucketName, objectName, nil) - err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) - if err != nil { - logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond failed", err) + if !bytes.Equal(decBytes, buf) { + logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) return } - err = src.SetMatchETagExceptCond(objInfo.ETag) + reader.Close() + + // Test key rotation for source object in-place. + newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"srcObject")) // replace key + dst, err = minio.NewDestinationInfo(bucketName, "srcObject", newSSE, nil) if err != nil { - logError(testName, function, args, startTime, "", "SetMatchETagExceptCond failed", err) + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } + args["destination"] = dst - // Perform the Copy which should fail err = c.CopyObject(dst, src) - if err == nil { - logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) return } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return - } - if err = cleanupBucket(bucketName+"-copy", c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) + // Get copied object and check if content is equal + reader, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) return } - successLogger(testName, function, args, startTime).Info() -} - -func testComposeObjectErrorCasesWrapper(c *minio.Client) { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "ComposeObject(destination, sourceList)" - args := map[string]interface{}{} - - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") + decBytes, err = ioutil.ReadAll(reader) if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + if !bytes.Equal(decBytes, buf) { + logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) return } + reader.Close() - // Test that more than 10K source objects cannot be - // concatenated. - srcArr := [10001]minio.SourceInfo{} - srcSlice := srcArr[:] - dst, err := minio.NewDestinationInfo(bucketName, "object", nil, nil) + // Test in-place decryption. + dst, err = minio.NewDestinationInfo(bucketName, "srcObject", nil, nil) if err != nil { logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } - args["destination"] = dst - // Just explain about srcArr in args["sourceList"] - // to stop having 10,001 null headers logged - args["sourceList"] = "source array of 10,001 elements" - if err := c.ComposeObject(dst, srcSlice); err == nil { - logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err) - return - } else if err.Error() != "There must be as least one and up to 10000 source objects." { - logError(testName, function, args, startTime, "", "Got unexpected error", err) + + src = minio.NewSourceInfo(bucketName, "srcObject", newSSE) + args["source"] = src + err = c.CopyObject(dst, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) return } - // Create a source with invalid offset spec and check that - // error is returned: - // 1. Create the source object. - const badSrcSize = 5 * 1024 * 1024 - buf := bytes.Repeat([]byte("1"), badSrcSize) - _, err = c.PutObject(bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) + // Get copied decrypted object and check if content is equal + reader, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{}) if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) + logError(testName, function, args, startTime, "", "GetObject failed", err) return } - // 2. Set invalid range spec on the object (going beyond - // object size) - badSrc := minio.NewSourceInfo(bucketName, "badObject", nil) - err = badSrc.SetRange(1, badSrcSize) + defer reader.Close() + + decBytes, err = ioutil.ReadAll(reader) if err != nil { - logError(testName, function, args, startTime, "", "Setting NewSourceInfo failed", err) + logError(testName, function, args, startTime, "", "ReadAll failed", err) return } - // 3. ComposeObject call should fail. - if err := c.ComposeObject(dst, []minio.SourceInfo{badSrc}); err == nil { - logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err) - return - } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { - logError(testName, function, args, startTime, "", "Got invalid error", err) + if !bytes.Equal(decBytes, buf) { + logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) return } @@ -5029,16 +5590,16 @@ func testComposeObjectErrorCasesWrapper(c *minio.Client) { successLogger(testName, function, args, startTime).Info() } -// Test expected error cases -func testComposeObjectErrorCasesV2() { +// Test encrypted copy object +func testEncryptedCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "ComposeObject(destination, sourceList)" + function := "CopyObject(destination, source)" args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV2( + c, err := minio.NewV4( os.Getenv(serverEndpoint), os.Getenv(accessKey), os.Getenv(secretKey), @@ -5049,230 +5610,235 @@ func testComposeObjectErrorCasesV2() { return } - testComposeObjectErrorCasesWrapper(c) + // c.TraceOn(os.Stderr) + testEncryptedCopyObjectWrapper(c) } -func testComposeMultipleSources(c *minio.Client) { +// Test encrypted copy object +func testEncryptedCopyObjectV2() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "ComposeObject(destination, sourceList)" - args := map[string]interface{}{ - "destination": "", - "sourceList": "", - } + function := "CopyObject(destination, source)" + args := map[string]interface{}{} - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") + // Instantiate new minio client object + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) if err != nil { - logError(testName, function, args, startTime, "", "MakeBucket failed", err) + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) return } - // Upload a small source object - const srcSize = 1024 * 1024 * 5 - buf := bytes.Repeat([]byte("1"), srcSize) - _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject failed", err) - return - } + // c.TraceOn(os.Stderr) + testEncryptedCopyObjectWrapper(c) +} - // We will append 10 copies of the object. - srcs := []minio.SourceInfo{} - for i := 0; i < 10; i++ { - srcs = append(srcs, minio.NewSourceInfo(bucketName, "srcObject", nil)) - } - // make the last part very small - err = srcs[9].SetRange(0, 0) +func testDecryptedCopyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) if err != nil { - logError(testName, function, args, startTime, "", "SetRange failed", err) + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) return } - args["sourceList"] = srcs - - dst, err := minio.NewDestinationInfo(bucketName, "dstObject", nil, nil) - args["destination"] = dst - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + bucketName, objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-"), "object" + if err = c.MakeBucket(bucketName, "us-east-1"); err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } - err = c.ComposeObject(dst, srcs) + + encryption := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)) + _, err = c.PutObject(bucketName, objectName, bytes.NewReader(bytes.Repeat([]byte("a"), 1024*1024)), 1024*1024, minio.PutObjectOptions{ + ServerSideEncryption: encryption, + }) if err != nil { - logError(testName, function, args, startTime, "", "ComposeObject failed", err) + logError(testName, function, args, startTime, "", "PutObject call failed", err) return } - objProps, err := c.StatObject(bucketName, "dstObject", minio.StatObjectOptions{}) + src := minio.NewSourceInfo(bucketName, objectName, encrypt.SSECopy(encryption)) + args["source"] = src + dst, err := minio.NewDestinationInfo(bucketName, "decrypted-"+objectName, nil, nil) if err != nil { - logError(testName, function, args, startTime, "", "StatObject failed", err) + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) return } + args["destination"] = dst - if objProps.Size != 9*srcSize+1 { - logError(testName, function, args, startTime, "", "Size mismatched! Expected "+string(10000*srcSize)+" got "+string(objProps.Size), err) + if err = c.CopyObject(dst, src); err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) return } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) + if _, err = c.GetObject(bucketName, "decrypted-"+objectName, minio.GetObjectOptions{}); err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) return } successLogger(testName, function, args, startTime).Info() } -// Test concatenating multiple objects objects -func testCompose10KSourcesV2() { +// Test Core CopyObjectPart implementation +func testCoreEncryptedCopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() - function := "ComposeObject(destination, sourceList)" + function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} // Instantiate new minio client object - c, err := minio.NewV2( + client, err := minio.NewV4( os.Getenv(serverEndpoint), os.Getenv(accessKey), os.Getenv(secretKey), mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) return } - testComposeMultipleSources(c) -} + // Instantiate new core client object. + c := minio.Core{client} -func testEncryptedCopyObjectWrapper(c *minio.Client) { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "CopyObject(destination, source)" - args := map[string]interface{}{} + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - // Make a new bucket in 'us-east-1' (source bucket). - err := c.MakeBucket(bucketName, "us-east-1") + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) - return } + defer cleanupBucket(bucketName, client) + // Make a buffer with 5MB of data + buf := bytes.Repeat([]byte("abcde"), 1024*1024) - key1 := minio.NewSSEInfo([]byte("32byteslongsecretkeymustbegiven1"), "AES256") - key2 := minio.NewSSEInfo([]byte("32byteslongsecretkeymustbegiven2"), "AES256") + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + password := "correct horse battery staple" + srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) - // 1. create an sse-c encrypted object to copy by uploading - const srcSize = 1024 * 1024 - buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB - metadata := make(map[string]string) - for k, v := range key1.GetSSEHeaders() { - metadata[k] = v - } - _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{UserMetadata: metadata, Progress: nil}) + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + "Content-Type": "binary/octet-stream", + }, srcencryption) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) - return } - // 2. copy object and change encryption key - src := minio.NewSourceInfo(bucketName, "srcObject", &key1) - args["source"] = src - dst, err := minio.NewDestinationInfo(bucketName, "dstObject", &key2, nil) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + if objInfo.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) } - args["destination"] = dst - err = c.CopyObject(dst, src) + destBucketName := bucketName + destObjectName := objectName + "-dest" + dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) + + uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { - logError(testName, function, args, startTime, "", "CopyObject failed", err) - return + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) } - // 3. get copied object and check if content is equal - opts := minio.GetObjectOptions{} - for k, v := range key2.GetSSEHeaders() { - opts.Set(k, v) + // Content of the destination object will be two copies of + // `objectName` concatenated, followed by first byte of + // `objectName`. + metadata := make(map[string]string) + header := make(http.Header) + encrypt.SSECopy(srcencryption).Marshal(header) + dstencryption.Marshal(header) + for k, v := range header { + metadata[k] = v[0] } - coreClient := minio.Core{c} - reader, _, err := coreClient.GetObject(bucketName, "dstObject", opts) + // First of three parts + fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) - return + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } - defer reader.Close() - decBytes, err := ioutil.ReadAll(reader) + // Second of three parts + sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) - return - } - if !bytes.Equal(decBytes, buf) { - logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) - return - } - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) - return + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) } - successLogger(testName, function, args, startTime).Info() -} + // Last of three parts + lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } -// Test encrypted copy object -func testEncryptedCopyObject() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "CopyObject(destination, source)" - args := map[string]interface{}{} + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } - // Instantiate new minio client object - c, err := minio.NewV4( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + // Stat the object and check its length matches + objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return + logError(testName, function, args, startTime, "", "StatObject call failed", err) } - // c.TraceOn(os.Stderr) - testEncryptedCopyObjectWrapper(c) -} + if objInfo.Size != (5*1024*1024)*2+1 { + logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) + } -// Test encrypted copy object -func testEncryptedCopyObjectV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "CopyObject(destination, source)" - args := map[string]interface{}{} + // Now we read the data back + getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} + getOpts.SetRange(0, 5*1024*1024-1) + r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf := make([]byte, 5*1024*1024) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf, buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) + } - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) + getOpts.SetRange(5*1024*1024, 0) + r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) - return + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf = make([]byte, 5*1024*1024+1) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf[:5*1024*1024], buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) + } + if getBuf[5*1024*1024] != buf[0] { + logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) } - testEncryptedCopyObjectWrapper(c) -} + successLogger(testName, function, args, startTime).Info() + // Do not need to remove destBucketName its same as bucketName. +} func testUserMetadataCopying() { // initialize logging params startTime := time.Now() @@ -5413,76 +5979,315 @@ func testUserMetadataCopyingWrapper(c *minio.Client) { args["destination"] = dst3 err = c.ComposeObject(dst3, srcs) if err != nil { - logError(testName, function, args, startTime, "", "ComposeObject failed", err) + logError(testName, function, args, startTime, "", "ComposeObject failed", err) + return + } + + // Check that no headers are copied in this case + if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) { + logError(testName, function, args, startTime, "", "Metadata match failed", err) + return + } + + // 7. Compose a pair of sources with dest user metadata set. + srcs = []minio.SourceInfo{ + minio.NewSourceInfo(bucketName, "srcObject", nil), + minio.NewSourceInfo(bucketName, "srcObject", nil), + } + dst4, err := minio.NewDestinationInfo(bucketName, "dstObject-4", nil, map[string]string{"notmyheader": "notmyvalue"}) + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + + function = "ComposeObject(destination, sources)" + args["source"] = srcs + args["destination"] = dst4 + err = c.ComposeObject(dst4, srcs) + if err != nil { + logError(testName, function, args, startTime, "", "ComposeObject failed", err) + return + } + + // Check that no headers are copied in this case + expectedHeaders = make(http.Header) + expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") + if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) { + logError(testName, function, args, startTime, "", "Metadata match failed", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testUserMetadataCopyingV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + return + } + + // c.TraceOn(os.Stderr) + testUserMetadataCopyingWrapper(c) +} + +func testStorageClassMetadataPutObject() { + // initialize logging params + startTime := time.Now() + function := "testStorageClassMetadataPutObject()" + args := map[string]interface{}{} + testName := getFuncName() + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + return + } + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + fetchMeta := func(object string) (h http.Header) { + objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + h = make(http.Header) + for k, vs := range objInfo.Metadata { + if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { + for _, v := range vs { + h.Add(k, v) + } + } + } + return h + } + + metadata := make(http.Header) + metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + + emptyMetadata := make(http.Header) + + const srcSize = 1024 * 1024 + buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB + + _, err = c.PutObject(bucketName, "srcObjectRRSClass", + bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + // Get the returned metadata + returnedMeta := fetchMeta("srcObjectRRSClass") + + // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) + if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { + logError(testName, function, args, startTime, "", "Metadata match failed", err) + return + } + + metadata = make(http.Header) + metadata.Set("x-amz-storage-class", "STANDARD") + + _, err = c.PutObject(bucketName, "srcObjectSSClass", + bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClass")) { + logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) + return + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +func testStorageClassInvalidMetadataPutObject() { + // initialize logging params + startTime := time.Now() + function := "testStorageClassInvalidMetadataPutObject()" + args := map[string]interface{}{} + testName := getFuncName() + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + return + } + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + const srcSize = 1024 * 1024 + buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB + + _, err = c.PutObject(bucketName, "srcObjectRRSClass", + bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "INVALID_STORAGE_CLASS"}) + if err == nil { + logError(testName, function, args, startTime, "", "PutObject with invalid storage class passed, was expected to fail", err) + return + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +func testStorageClassMetadataCopyObject() { + // initialize logging params + startTime := time.Now() + function := "testStorageClassMetadataCopyObject()" + args := map[string]interface{}{} + testName := getFuncName() + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + return + } + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + fetchMeta := func(object string) (h http.Header) { + objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + h = make(http.Header) + for k, vs := range objInfo.Metadata { + if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { + for _, v := range vs { + h.Add(k, v) + } + } + } + return h + } + + metadata := make(http.Header) + metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + + emptyMetadata := make(http.Header) + + const srcSize = 1024 * 1024 + buf := bytes.Repeat([]byte("abcde"), srcSize) + + // Put an object with RRS Storage class + _, err = c.PutObject(bucketName, "srcObjectRRSClass", + bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) return } - // Check that no headers are copied in this case - if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) { + // Make server side copy of object uploaded in previous step + src := minio.NewSourceInfo(bucketName, "srcObjectRRSClass", nil) + dst, err := minio.NewDestinationInfo(bucketName, "srcObjectRRSClassCopy", nil, nil) + c.CopyObject(dst, src) + + // Get the returned metadata + returnedMeta := fetchMeta("srcObjectRRSClassCopy") + + // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) + if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } - // 7. Compose a pair of sources with dest user metadata set. - srcs = []minio.SourceInfo{ - minio.NewSourceInfo(bucketName, "srcObject", nil), - minio.NewSourceInfo(bucketName, "srcObject", nil), - } - dst4, err := minio.NewDestinationInfo(bucketName, "dstObject-4", nil, map[string]string{"notmyheader": "notmyvalue"}) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return - } + metadata = make(http.Header) + metadata.Set("x-amz-storage-class", "STANDARD") - function = "ComposeObject(destination, sources)" - args["source"] = srcs - args["destination"] = dst4 - err = c.ComposeObject(dst4, srcs) + // Put an object with Standard Storage class + _, err = c.PutObject(bucketName, "srcObjectSSClass", + bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) if err != nil { - logError(testName, function, args, startTime, "", "ComposeObject failed", err) + logError(testName, function, args, startTime, "", "PutObject failed", err) return } - // Check that no headers are copied in this case - expectedHeaders = make(http.Header) - expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") - if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) { - logError(testName, function, args, startTime, "", "Metadata match failed", err) + // Make server side copy of object uploaded in previous step + src = minio.NewSourceInfo(bucketName, "srcObjectSSClass", nil) + dst, err = minio.NewDestinationInfo(bucketName, "srcObjectSSClassCopy", nil, nil) + c.CopyObject(dst, src) + + // Fetch the meta data of copied object + if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClassCopy")) { + logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) return } - // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) return } - successLogger(testName, function, args, startTime).Info() } -func testUserMetadataCopyingV2() { - // initialize logging params - startTime := time.Now() - testName := getFuncName() - function := "CopyObject(destination, source)" - args := map[string]interface{}{} - - // Instantiate new minio client object - c, err := minio.NewV2( - os.Getenv(serverEndpoint), - os.Getenv(accessKey), - os.Getenv(secretKey), - mustParseBool(os.Getenv(enableHTTPS)), - ) - if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) - return - } - - // c.TraceOn(os.Stderr) - testUserMetadataCopyingWrapper(c) -} - // Test put object with size -1 byte object. func testPutObjectNoLengthV2() { // initialize logging params @@ -5763,7 +6568,7 @@ func testFunctionalV2() { startTime := time.Now() testName := getFuncName() function := "testFunctionalV2()" - function_all := "" + functionAll := "" args := map[string]interface{}{} // Seed random based on current time. @@ -5791,7 +6596,7 @@ func testFunctionalV2() { location := "us-east-1" // Make a new bucket. function = "MakeBucket(bucketName, location)" - function_all = "MakeBucket(bucketName, location)" + functionAll = "MakeBucket(bucketName, location)" args = map[string]interface{}{ "bucketName": bucketName, "location": location, @@ -5822,7 +6627,7 @@ func testFunctionalV2() { // Verify if bucket exits and you have access. var exists bool function = "BucketExists(bucketName)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } @@ -5837,14 +6642,17 @@ func testFunctionalV2() { } // Make the bucket 'public read/write'. - function = "SetBucketPolicy(bucketName, objectPrefix, bucketPolicy)" - function_all += ", " + function + function = "SetBucketPolicy(bucketName, bucketPolicy)" + functionAll += ", " + function + + readWritePolicy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:ListBucketMultipartUploads", "s3:ListBucket"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::` + bucketName + `"],"Sid": ""}]}` + args = map[string]interface{}{ "bucketName": bucketName, - "objectPrefix": "", - "bucketPolicy": policy.BucketPolicyReadWrite, + "bucketPolicy": readWritePolicy, } - err = c.SetBucketPolicy(bucketName, "", policy.BucketPolicyReadWrite) + err = c.SetBucketPolicy(bucketName, readWritePolicy) + if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return @@ -5852,7 +6660,7 @@ func testFunctionalV2() { // List all buckets. function = "ListBuckets()" - function_all += ", " + function + functionAll += ", " + function args = nil buckets, err := c.ListBuckets() if len(buckets) == 0 { @@ -5898,9 +6706,9 @@ func testFunctionalV2() { return } - objectName_noLength := objectName + "-nolength" - args["objectName"] = objectName_noLength - n, err = c.PutObject(bucketName, objectName_noLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + objectNameNoLength := objectName + "-nolength" + args["objectName"] = objectNameNoLength + n, err = c.PutObject(bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return @@ -5918,7 +6726,7 @@ func testFunctionalV2() { objFound := false isRecursive := true // Recursive is true. function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -5937,7 +6745,7 @@ func testFunctionalV2() { incompObjNotFound := true function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -5955,7 +6763,7 @@ func testFunctionalV2() { } function = "GetObject(bucketName, objectName)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -5971,6 +6779,7 @@ func testFunctionalV2() { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } + newReader.Close() if !bytes.Equal(newReadBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch", err) @@ -5978,7 +6787,7 @@ func testFunctionalV2() { } function = "FGetObject(bucketName, objectName, fileName)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -5992,7 +6801,7 @@ func testFunctionalV2() { // Generate presigned HEAD object url. function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -6021,7 +6830,7 @@ func testFunctionalV2() { // Generate presigned GET object url. function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, @@ -6089,7 +6898,7 @@ func testFunctionalV2() { } function = "PresignedPutObject(bucketName, objectName, expires)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-presigned", @@ -6123,7 +6932,7 @@ func testFunctionalV2() { } function = "GetObject(bucketName, objectName)" - function_all += ", " + function + functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-presigned", @@ -6139,6 +6948,7 @@ func testFunctionalV2() { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } + newReader.Close() if !bytes.Equal(newReadBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch", err) @@ -6159,7 +6969,7 @@ func testFunctionalV2() { logError(testName, function, args, startTime, "", "File removes failed", err) return } - successLogger(testName, function_all, args, startTime).Info() + successLogger(testName, functionAll, args, startTime).Info() } // Test get object with GetObjectWithContext @@ -6227,10 +7037,12 @@ func testGetObjectWithContext() { logError(testName, function, args, startTime, "", "GetObjectWithContext failed unexpectedly", err) return } + if _, err = r.Stat(); err == nil { logError(testName, function, args, startTime, "", "GetObjectWithContext should fail on short timeout", err) return } + r.Close() ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) args["ctx"] = ctx @@ -6359,6 +7171,167 @@ func testFGetObjectWithContext() { } +// Test get object ACLs with GetObjectACL +func testGetObjectACL() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObjectACL(bucketName, objectName)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // skipping region functional tests for non s3 runs + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() + return + } + + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + bufSize := dataFileMap["datafile-1-MB"] + var reader = getDataReader("datafile-1-MB") + defer reader.Close() + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Add meta data to add a canned acl + metaData := map[string]string{ + "X-Amz-Acl": "public-read-write", + } + + _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + // Read the data back + objectInfo, getObjectACLErr := c.GetObjectACL(bucketName, objectName) + if getObjectACLErr == nil { + logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr) + return + } + + s, ok := objectInfo.Metadata["X-Amz-Acl"] + if !ok { + logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Acl\"", nil) + return + } + + if len(s) != 1 { + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) + return + } + + if s[0] != "public-read-write" { + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil) + return + } + + bufSize = dataFileMap["datafile-1-MB"] + var reader2 = getDataReader("datafile-1-MB") + defer reader2.Close() + // Save the data + objectName = randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Add meta data to add a canned acl + metaData = map[string]string{ + "X-Amz-Grant-Read": "id=fooread@minio.go", + "X-Amz-Grant-Write": "id=foowrite@minio.go", + } + + _, err = c.PutObject(bucketName, objectName, reader2, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + // Read the data back + objectInfo, getObjectACLErr = c.GetObjectACL(bucketName, objectName) + if getObjectACLErr == nil { + logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr) + return + } + + if len(objectInfo.Metadata) != 3 { + logError(testName, function, args, startTime, "", "GetObjectACL fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil) + return + } + + s, ok = objectInfo.Metadata["X-Amz-Grant-Read"] + if !ok { + logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Read\"", nil) + return + } + + if len(s) != 1 { + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) + return + } + + if s[0] != "fooread@minio.go" { + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil) + return + } + + s, ok = objectInfo.Metadata["X-Amz-Grant-Write"] + if !ok { + logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Write\"", nil) + return + } + + if len(s) != 1 { + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) + return + } + + if s[0] != "foowrite@minio.go" { + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + // Test validates putObject with context to see if request cancellation is honored for V2. func testPutObjectWithContextV2() { // initialize logging params @@ -6509,6 +7482,7 @@ func testGetObjectWithContextV2() { logError(testName, function, args, startTime, "", "GetObjectWithContext should fail on short timeout", err) return } + r.Close() ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() @@ -6638,6 +7612,120 @@ func testFGetObjectWithContextV2() { } +// Test list object v1 and V2 storage class fields +func testListObjects() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "ListObjects(bucketName, objectPrefix, recursive, doneCh)" + args := map[string]interface{}{ + "bucketName": "", + "objectPrefix": "", + "recursive": "true", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + // Save the data + objectName1 := randString(60, rand.NewSource(time.Now().UnixNano()), "") + + _, err = c.PutObject(bucketName, objectName1, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: "STANDARD"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject1 call failed", err) + return + } + + bufSize1 := dataFileMap["datafile-33-kB"] + var reader1 = getDataReader("datafile-33-kB") + defer reader1.Close() + objectName2 := randString(60, rand.NewSource(time.Now().UnixNano()), "") + + _, err = c.PutObject(bucketName, objectName2, reader1, int64(bufSize1), minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: "REDUCED_REDUNDANCY"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject2 call failed", err) + return + } + + // Create a done channel to control 'ListObjects' go routine. + doneCh := make(chan struct{}) + // Exit cleanly upon return. + defer close(doneCh) + + // check for storage-class from ListObjects result + for objInfo := range c.ListObjects(bucketName, "", true, doneCh) { + if objInfo.Err != nil { + logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err) + return + } + if objInfo.Key == objectName1 && objInfo.StorageClass != "STANDARD" { + // Ignored as Gateways (Azure/GCS etc) wont return storage class + ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info() + } + if objInfo.Key == objectName2 && objInfo.StorageClass != "REDUCED_REDUNDANCY" { + // Ignored as Gateways (Azure/GCS etc) wont return storage class + ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info() + } + } + + // check for storage-class from ListObjectsV2 result + for objInfo := range c.ListObjectsV2(bucketName, "", true, doneCh) { + if objInfo.Err != nil { + logError(testName, function, args, startTime, "", "ListObjectsV2 failed unexpectedly", err) + return + } + if objInfo.Key == objectName1 && objInfo.StorageClass != "STANDARD" { + // Ignored as Gateways (Azure/GCS etc) wont return storage class + ignoredLog(testName, function, args, startTime, "ListObjectsV2 doesn't return expected storage class").Info() + } + if objInfo.Key == objectName2 && objInfo.StorageClass != "REDUCED_REDUNDANCY" { + // Ignored as Gateways (Azure/GCS etc) wont return storage class + ignoredLog(testName, function, args, startTime, "ListObjectsV2 doesn't return expected storage class").Info() + } + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() + +} + // Convert string to bool and always return false if any error func mustParseBool(str string) bool { b, err := strconv.ParseBool(str) @@ -6659,10 +7747,9 @@ func main() { tls := mustParseBool(os.Getenv(enableHTTPS)) // execute tests - if !isQuickMode() { + if isFullMode() { testMakeBucketErrorV2() testGetObjectClosedTwiceV2() - testRemovePartiallyUploadedV2() testFPutObjectV2() testMakeBucketRegionsV2() testGetObjectReadSeekFunctionalV2() @@ -6684,19 +7771,15 @@ func main() { testPutObjectWithMetadata() testPutObjectReadAt() testPutObjectStreaming() - testListPartiallyUploaded() testGetObjectSeekEnd() testGetObjectClosedTwice() testRemoveMultipleObjects() - testRemovePartiallyUploaded() testFPutObjectMultipart() testFPutObject() testGetObjectReadSeekFunctional() testGetObjectReadAtFunctional() testPresignedPostPolicy() testCopyObject() - testEncryptionPutGet() - testEncryptionFPut() testComposeObjectErrorCases() testCompose10KSources() testUserMetadataCopying() @@ -6707,12 +7790,25 @@ func main() { testGetObjectWithContext() testFPutObjectWithContext() testFGetObjectWithContext() + testGetObjectACL() testPutObjectWithContext() + testStorageClassMetadataPutObject() + testStorageClassInvalidMetadataPutObject() + testStorageClassMetadataCopyObject() + testPutObjectWithContentLanguage() + testListObjects() // SSE-C tests will only work over TLS connection. if tls { + testEncryptionPutGet() + testEncryptionFPut() + testEncryptedGetObjectReadAtFunctional() + testEncryptedGetObjectReadSeekFunctional() testEncryptedCopyObjectV2() testEncryptedCopyObject() + testEncryptedEmptyObject() + testDecryptedCopyObject() + testCoreEncryptedCopyObjectPart() } } else { testFunctional() diff --git a/pkg/credentials/file_minio_client.go b/pkg/credentials/file_minio_client.go index c282c2a..6a6827e 100644 --- a/pkg/credentials/file_minio_client.go +++ b/pkg/credentials/file_minio_client.go @@ -62,13 +62,17 @@ func NewFileMinioClient(filename string, alias string) *Credentials { // users home directory. func (p *FileMinioClient) Retrieve() (Value, error) { if p.filename == "" { - homeDir, err := homedir.Dir() - if err != nil { - return Value{}, err - } - p.filename = filepath.Join(homeDir, ".mc", "config.json") - if runtime.GOOS == "windows" { - p.filename = filepath.Join(homeDir, "mc", "config.json") + if value, ok := os.LookupEnv("MINIO_SHARED_CREDENTIALS_FILE"); ok { + p.filename = value + } else { + homeDir, err := homedir.Dir() + if err != nil { + return Value{}, err + } + p.filename = filepath.Join(homeDir, ".mc", "config.json") + if runtime.GOOS == "windows" { + p.filename = filepath.Join(homeDir, "mc", "config.json") + } } } diff --git a/pkg/credentials/iam_aws.go b/pkg/credentials/iam_aws.go index 637df74..6845c9a 100644 --- a/pkg/credentials/iam_aws.go +++ b/pkg/credentials/iam_aws.go @@ -21,8 +21,10 @@ import ( "bufio" "encoding/json" "errors" + "fmt" "net/http" "net/url" + "os" "path" "time" ) @@ -50,16 +52,25 @@ type IAM struct { // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html const ( defaultIAMRoleEndpoint = "http://169.254.169.254" + defaultECSRoleEndpoint = "http://169.254.170.2" defaultIAMSecurityCredsPath = "/latest/meta-data/iam/security-credentials" ) +// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html +func getEndpoint(endpoint string) (string, bool) { + if endpoint != "" { + return endpoint, os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") != "" + } + if ecsURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"); ecsURI != "" { + return fmt.Sprintf("%s%s", defaultECSRoleEndpoint, ecsURI), true + } + return defaultIAMRoleEndpoint, false +} + // NewIAM returns a pointer to a new Credentials object wrapping // the IAM. Takes a ConfigProvider to create a EC2Metadata client. // The ConfigProvider is satisfied by the session.Session type. func NewIAM(endpoint string) *Credentials { - if endpoint == "" { - endpoint = defaultIAMRoleEndpoint - } p := &IAM{ Client: &http.Client{ Transport: http.DefaultTransport, @@ -73,11 +84,17 @@ func NewIAM(endpoint string) *Credentials { // Error will be returned if the request fails, or unable to extract // the desired func (m *IAM) Retrieve() (Value, error) { - roleCreds, err := getCredentials(m.Client, m.endpoint) + endpoint, isEcsTask := getEndpoint(m.endpoint) + var roleCreds ec2RoleCredRespBody + var err error + if isEcsTask { + roleCreds, err = getEcsTaskCredentials(m.Client, endpoint) + } else { + roleCreds, err = getCredentials(m.Client, endpoint) + } if err != nil { return Value{}, err } - // Expiry window is set to 10secs. m.SetExpiration(roleCreds.Expiration, DefaultExpiryWindow) @@ -111,9 +128,6 @@ type ec2RoleCredRespBody struct { // be sent to fetch the rolling access credentials. // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html func getIAMRoleURL(endpoint string) (*url.URL, error) { - if endpoint == "" { - endpoint = defaultIAMRoleEndpoint - } u, err := url.Parse(endpoint) if err != nil { return nil, err @@ -153,12 +167,36 @@ func listRoleNames(client *http.Client, u *url.URL) ([]string, error) { return credsList, nil } +func getEcsTaskCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, error) { + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return ec2RoleCredRespBody{}, err + } + + resp, err := client.Do(req) + if err != nil { + return ec2RoleCredRespBody{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return ec2RoleCredRespBody{}, errors.New(resp.Status) + } + + respCreds := ec2RoleCredRespBody{} + if err := json.NewDecoder(resp.Body).Decode(&respCreds); err != nil { + return ec2RoleCredRespBody{}, err + } + + return respCreds, nil +} + // getCredentials - obtains the credentials from the IAM role name associated with // the current EC2 service. // // If the credentials cannot be found, or there is an error // reading the response an error will be returned. func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, error) { + // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html u, err := getIAMRoleURL(endpoint) if err != nil { diff --git a/pkg/credentials/iam_aws_test.go b/pkg/credentials/iam_aws_test.go index 86ea66b..4dbbb0a 100644 --- a/pkg/credentials/iam_aws_test.go +++ b/pkg/credentials/iam_aws_test.go @@ -21,6 +21,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "os" "testing" "time" ) @@ -41,6 +42,13 @@ const credsFailRespTmpl = `{ "LastUpdated": "2009-11-23T0:00:00Z" }` +const credsRespEcsTaskTmpl = `{ + "AccessKeyId" : "accessKey", + "SecretAccessKey" : "secret", + "Token" : "token", + "Expiration" : "%s" +}` + func initTestFailServer() *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "Not allowed", http.StatusBadRequest) @@ -73,6 +81,14 @@ func initTestServer(expireOn string, failAssume bool) *httptest.Server { return server } +func initEcsTaskTestServer(expireOn string) *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, credsRespEcsTaskTmpl, expireOn) + })) + + return server +} + func TestIAMMalformedEndpoint(t *testing.T) { creds := NewIAM("%%%%") _, err := creds.Get() @@ -195,3 +211,33 @@ func TestIAMIsExpired(t *testing.T) { t.Error("Expected creds to be expired when curren time has changed") } } + +func TestEcsTask(t *testing.T) { + server := initEcsTaskTestServer("2014-12-16T01:51:37Z") + defer server.Close() + p := &IAM{ + Client: http.DefaultClient, + endpoint: server.URL, + } + os.Setenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "/v2/credentials?id=task_credential_id") + creds, err := p.Retrieve() + os.Unsetenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + if err != nil { + t.Errorf("Unexpected failure %s", err) + } + if "accessKey" != creds.AccessKeyID { + t.Errorf("Expected \"accessKey\", got %s", creds.AccessKeyID) + } + + if "secret" != creds.SecretAccessKey { + t.Errorf("Expected \"secret\", got %s", creds.SecretAccessKey) + } + + if "token" != creds.SessionToken { + t.Errorf("Expected \"token\", got %s", creds.SessionToken) + } + + if !p.IsExpired() { + t.Error("Expected creds to be expired.") + } +} diff --git a/pkg/encrypt/cbc.go b/pkg/encrypt/cbc.go deleted file mode 100644 index b0f2d6e..0000000 --- a/pkg/encrypt/cbc.go +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package encrypt - -import ( - "bytes" - "crypto/aes" - "crypto/cipher" - "crypto/rand" - "encoding/base64" - "errors" - "io" -) - -// Crypt mode - encryption or decryption -type cryptMode int - -const ( - encryptMode cryptMode = iota - decryptMode -) - -// CBCSecureMaterials encrypts/decrypts data using AES CBC algorithm -type CBCSecureMaterials struct { - - // Data stream to encrypt/decrypt - stream io.Reader - - // Last internal error - err error - - // End of file reached - eof bool - - // Holds initial data - srcBuf *bytes.Buffer - - // Holds transformed data (encrypted or decrypted) - dstBuf *bytes.Buffer - - // Encryption algorithm - encryptionKey Key - - // Key to encrypts/decrypts data - contentKey []byte - - // Encrypted form of contentKey - cryptedKey []byte - - // Initialization vector - iv []byte - - // matDesc - currently unused - matDesc []byte - - // Indicate if we are going to encrypt or decrypt - cryptMode cryptMode - - // Helper that encrypts/decrypts data - blockMode cipher.BlockMode -} - -// NewCBCSecureMaterials builds new CBC crypter module with -// the specified encryption key (symmetric or asymmetric) -func NewCBCSecureMaterials(key Key) (*CBCSecureMaterials, error) { - if key == nil { - return nil, errors.New("Unable to recognize empty encryption properties") - } - return &CBCSecureMaterials{ - srcBuf: bytes.NewBuffer([]byte{}), - dstBuf: bytes.NewBuffer([]byte{}), - encryptionKey: key, - matDesc: []byte("{}"), - }, nil - -} - -// Close implements closes the internal stream. -func (s *CBCSecureMaterials) Close() error { - closer, ok := s.stream.(io.Closer) - if ok { - return closer.Close() - } - return nil -} - -// SetupEncryptMode - tells CBC that we are going to encrypt data -func (s *CBCSecureMaterials) SetupEncryptMode(stream io.Reader) error { - // Set mode to encrypt - s.cryptMode = encryptMode - - // Set underlying reader - s.stream = stream - - s.eof = false - s.srcBuf.Reset() - s.dstBuf.Reset() - - var err error - - // Generate random content key - s.contentKey = make([]byte, aes.BlockSize*2) - if _, err := rand.Read(s.contentKey); err != nil { - return err - } - // Encrypt content key - s.cryptedKey, err = s.encryptionKey.Encrypt(s.contentKey) - if err != nil { - return err - } - // Generate random IV - s.iv = make([]byte, aes.BlockSize) - if _, err = rand.Read(s.iv); err != nil { - return err - } - // New cipher - encryptContentBlock, err := aes.NewCipher(s.contentKey) - if err != nil { - return err - } - - s.blockMode = cipher.NewCBCEncrypter(encryptContentBlock, s.iv) - - return nil -} - -// SetupDecryptMode - tells CBC that we are going to decrypt data -func (s *CBCSecureMaterials) SetupDecryptMode(stream io.Reader, iv string, key string) error { - // Set mode to decrypt - s.cryptMode = decryptMode - - // Set underlying reader - s.stream = stream - - // Reset - s.eof = false - s.srcBuf.Reset() - s.dstBuf.Reset() - - var err error - - // Get IV - s.iv, err = base64.StdEncoding.DecodeString(iv) - if err != nil { - return err - } - - // Get encrypted content key - s.cryptedKey, err = base64.StdEncoding.DecodeString(key) - if err != nil { - return err - } - - // Decrypt content key - s.contentKey, err = s.encryptionKey.Decrypt(s.cryptedKey) - if err != nil { - return err - } - - // New cipher - decryptContentBlock, err := aes.NewCipher(s.contentKey) - if err != nil { - return err - } - - s.blockMode = cipher.NewCBCDecrypter(decryptContentBlock, s.iv) - return nil -} - -// GetIV - return randomly generated IV (per S3 object), base64 encoded. -func (s *CBCSecureMaterials) GetIV() string { - return base64.StdEncoding.EncodeToString(s.iv) -} - -// GetKey - return content encrypting key (cek) in encrypted form, base64 encoded. -func (s *CBCSecureMaterials) GetKey() string { - return base64.StdEncoding.EncodeToString(s.cryptedKey) -} - -// GetDesc - user provided encryption material description in JSON (UTF8) format. -func (s *CBCSecureMaterials) GetDesc() string { - return string(s.matDesc) -} - -// Fill buf with encrypted/decrypted data -func (s *CBCSecureMaterials) Read(buf []byte) (n int, err error) { - // Always fill buf from bufChunk at the end of this function - defer func() { - if s.err != nil { - n, err = 0, s.err - } else { - n, err = s.dstBuf.Read(buf) - } - }() - - // Return - if s.eof { - return - } - - // Fill dest buffer if its length is less than buf - for !s.eof && s.dstBuf.Len() < len(buf) { - - srcPart := make([]byte, aes.BlockSize) - dstPart := make([]byte, aes.BlockSize) - - // Fill src buffer - for s.srcBuf.Len() < aes.BlockSize*2 { - _, err = io.CopyN(s.srcBuf, s.stream, aes.BlockSize) - if err != nil { - break - } - } - - // Quit immediately for errors other than io.EOF - if err != nil && err != io.EOF { - s.err = err - return - } - - // Mark current encrypting/decrypting as finished - s.eof = (err == io.EOF) - - if s.eof && s.cryptMode == encryptMode { - if srcPart, err = pkcs5Pad(s.srcBuf.Bytes(), aes.BlockSize); err != nil { - s.err = err - return - } - } else { - _, _ = s.srcBuf.Read(srcPart) - } - - // Crypt srcPart content - for len(srcPart) > 0 { - - // Crypt current part - s.blockMode.CryptBlocks(dstPart, srcPart[:aes.BlockSize]) - - // Unpad when this is the last part and we are decrypting - if s.eof && s.cryptMode == decryptMode { - dstPart, err = pkcs5Unpad(dstPart, aes.BlockSize) - if err != nil { - s.err = err - return - } - } - - // Send crypted data to dstBuf - if _, wErr := s.dstBuf.Write(dstPart); wErr != nil { - s.err = wErr - return - } - // Move to the next part - srcPart = srcPart[aes.BlockSize:] - } - } - return -} - -// Unpad a set of bytes following PKCS5 algorithm -func pkcs5Unpad(buf []byte, blockSize int) ([]byte, error) { - len := len(buf) - if len == 0 { - return nil, errors.New("buffer is empty") - } - pad := int(buf[len-1]) - if pad > len || pad > blockSize { - return nil, errors.New("invalid padding size") - } - return buf[:len-pad], nil -} - -// Pad a set of bytes following PKCS5 algorithm -func pkcs5Pad(buf []byte, blockSize int) ([]byte, error) { - len := len(buf) - pad := blockSize - (len % blockSize) - padText := bytes.Repeat([]byte{byte(pad)}, pad) - return append(buf, padText...), nil -} diff --git a/pkg/encrypt/interface.go b/pkg/encrypt/interface.go deleted file mode 100644 index 482922a..0000000 --- a/pkg/encrypt/interface.go +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package encrypt implements a generic interface to encrypt any stream of data. -// currently this package implements two types of encryption -// - Symmetric encryption using AES. -// - Asymmetric encrytion using RSA. -package encrypt - -import "io" - -// Materials - provides generic interface to encrypt any stream of data. -type Materials interface { - - // Closes the wrapped stream properly, initiated by the caller. - Close() error - - // Returns encrypted/decrypted data, io.Reader compatible. - Read(b []byte) (int, error) - - // Get randomly generated IV, base64 encoded. - GetIV() (iv string) - - // Get content encrypting key (cek) in encrypted form, base64 encoded. - GetKey() (key string) - - // Get user provided encryption material description in - // JSON (UTF8) format. This is not used, kept for future. - GetDesc() (desc string) - - // Setup encrypt mode, further calls of Read() function - // will return the encrypted form of data streamed - // by the passed reader - SetupEncryptMode(stream io.Reader) error - - // Setup decrypted mode, further calls of Read() function - // will return the decrypted form of data streamed - // by the passed reader - SetupDecryptMode(stream io.Reader, iv string, key string) error -} diff --git a/pkg/encrypt/keys.go b/pkg/encrypt/keys.go deleted file mode 100644 index 0ed95f5..0000000 --- a/pkg/encrypt/keys.go +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package encrypt - -import ( - "crypto/aes" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "errors" -) - -// Key - generic interface to encrypt/decrypt a key. -// We use it to encrypt/decrypt content key which is the key -// that encrypt/decrypt object data. -type Key interface { - // Encrypt data using to the set encryption key - Encrypt([]byte) ([]byte, error) - // Decrypt data using to the set encryption key - Decrypt([]byte) ([]byte, error) -} - -// SymmetricKey - encrypts data with a symmetric master key -type SymmetricKey struct { - masterKey []byte -} - -// Encrypt passed bytes -func (s *SymmetricKey) Encrypt(plain []byte) ([]byte, error) { - // Initialize an AES encryptor using a master key - keyBlock, err := aes.NewCipher(s.masterKey) - if err != nil { - return []byte{}, err - } - - // Pad the key before encryption - plain, _ = pkcs5Pad(plain, aes.BlockSize) - - encKey := []byte{} - encPart := make([]byte, aes.BlockSize) - - // Encrypt the passed key by block - for { - if len(plain) < aes.BlockSize { - break - } - // Encrypt the passed key - keyBlock.Encrypt(encPart, plain[:aes.BlockSize]) - // Add the encrypted block to the total encrypted key - encKey = append(encKey, encPart...) - // Pass to the next plain block - plain = plain[aes.BlockSize:] - } - return encKey, nil -} - -// Decrypt passed bytes -func (s *SymmetricKey) Decrypt(cipher []byte) ([]byte, error) { - // Initialize AES decrypter - keyBlock, err := aes.NewCipher(s.masterKey) - if err != nil { - return nil, err - } - - var plain []byte - plainPart := make([]byte, aes.BlockSize) - - // Decrypt the encrypted data block by block - for { - if len(cipher) < aes.BlockSize { - break - } - keyBlock.Decrypt(plainPart, cipher[:aes.BlockSize]) - // Add the decrypted block to the total result - plain = append(plain, plainPart...) - // Pass to the next cipher block - cipher = cipher[aes.BlockSize:] - } - - // Unpad the resulted plain data - plain, err = pkcs5Unpad(plain, aes.BlockSize) - if err != nil { - return nil, err - } - - return plain, nil -} - -// NewSymmetricKey generates a new encrypt/decrypt crypto using -// an AES master key password -func NewSymmetricKey(b []byte) *SymmetricKey { - return &SymmetricKey{masterKey: b} -} - -// AsymmetricKey - struct which encrypts/decrypts data -// using RSA public/private certificates -type AsymmetricKey struct { - publicKey *rsa.PublicKey - privateKey *rsa.PrivateKey -} - -// Encrypt data using public key -func (a *AsymmetricKey) Encrypt(plain []byte) ([]byte, error) { - cipher, err := rsa.EncryptPKCS1v15(rand.Reader, a.publicKey, plain) - if err != nil { - return nil, err - } - return cipher, nil -} - -// Decrypt data using public key -func (a *AsymmetricKey) Decrypt(cipher []byte) ([]byte, error) { - cipher, err := rsa.DecryptPKCS1v15(rand.Reader, a.privateKey, cipher) - if err != nil { - return nil, err - } - return cipher, nil -} - -// NewAsymmetricKey - generates a crypto module able to encrypt/decrypt -// data using a pair for private and public key -func NewAsymmetricKey(privData []byte, pubData []byte) (*AsymmetricKey, error) { - // Parse private key from passed data - priv, err := x509.ParsePKCS8PrivateKey(privData) - if err != nil { - return nil, err - } - privKey, ok := priv.(*rsa.PrivateKey) - if !ok { - return nil, errors.New("not a valid private key") - } - - // Parse public key from passed data - pub, err := x509.ParsePKIXPublicKey(pubData) - if err != nil { - return nil, err - } - - pubKey, ok := pub.(*rsa.PublicKey) - if !ok { - return nil, errors.New("not a valid public key") - } - - // Associate the private key with the passed public key - privKey.PublicKey = *pubKey - - return &AsymmetricKey{ - publicKey: pubKey, - privateKey: privKey, - }, nil -} diff --git a/pkg/encrypt/server-side.go b/pkg/encrypt/server-side.go new file mode 100644 index 0000000..2d3c70f --- /dev/null +++ b/pkg/encrypt/server-side.go @@ -0,0 +1,195 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package encrypt + +import ( + "crypto/md5" + "encoding/base64" + "encoding/json" + "errors" + "net/http" + + "golang.org/x/crypto/argon2" +) + +const ( + // sseGenericHeader is the AWS SSE header used for SSE-S3 and SSE-KMS. + sseGenericHeader = "X-Amz-Server-Side-Encryption" + + // sseKmsKeyID is the AWS SSE-KMS key id. + sseKmsKeyID = sseGenericHeader + "-Aws-Kms-Key-Id" + // sseEncryptionContext is the AWS SSE-KMS Encryption Context data. + sseEncryptionContext = sseGenericHeader + "-Encryption-Context" + + // sseCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key. + sseCustomerAlgorithm = sseGenericHeader + "-Customer-Algorithm" + // sseCustomerKey is the AWS SSE-C encryption key HTTP header key. + sseCustomerKey = sseGenericHeader + "-Customer-Key" + // sseCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key. + sseCustomerKeyMD5 = sseGenericHeader + "-Customer-Key-MD5" + + // sseCopyCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key for CopyObject API. + sseCopyCustomerAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm" + // sseCopyCustomerKey is the AWS SSE-C encryption key HTTP header key for CopyObject API. + sseCopyCustomerKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key" + // sseCopyCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key for CopyObject API. + sseCopyCustomerKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-MD5" +) + +// PBKDF creates a SSE-C key from the provided password and salt. +// PBKDF is a password-based key derivation function +// which can be used to derive a high-entropy cryptographic +// key from a low-entropy password and a salt. +type PBKDF func(password, salt []byte) ServerSide + +// DefaultPBKDF is the default PBKDF. It uses Argon2id with the +// recommended parameters from the RFC draft (1 pass, 64 MB memory, 4 threads). +var DefaultPBKDF PBKDF = func(password, salt []byte) ServerSide { + sse := ssec{} + copy(sse[:], argon2.IDKey(password, salt, 1, 64*1024, 4, 32)) + return sse +} + +// Type is the server-side-encryption method. It represents one of +// the following encryption methods: +// - SSE-C: server-side-encryption with customer provided keys +// - KMS: server-side-encryption with managed keys +// - S3: server-side-encryption using S3 storage encryption +type Type string + +const ( + // SSEC represents server-side-encryption with customer provided keys + SSEC Type = "SSE-C" + // KMS represents server-side-encryption with managed keys + KMS Type = "KMS" + // S3 represents server-side-encryption using S3 storage encryption + S3 Type = "S3" +) + +// ServerSide is a form of S3 server-side-encryption. +type ServerSide interface { + // Type returns the server-side-encryption method. + Type() Type + + // Marshal adds encryption headers to the provided HTTP headers. + // It marks an HTTP request as server-side-encryption request + // and inserts the required data into the headers. + Marshal(h http.Header) +} + +// NewSSE returns a server-side-encryption using S3 storage encryption. +// Using SSE-S3 the server will encrypt the object with server-managed keys. +func NewSSE() ServerSide { return s3{} } + +// NewSSEKMS returns a new server-side-encryption using SSE-KMS and the provided Key Id and context. +func NewSSEKMS(keyID string, context interface{}) (ServerSide, error) { + if context == nil { + return kms{key: keyID, hasContext: false}, nil + } + serializedContext, err := json.Marshal(context) + if err != nil { + return nil, err + } + return kms{key: keyID, context: serializedContext, hasContext: true}, nil +} + +// NewSSEC returns a new server-side-encryption using SSE-C and the provided key. +// The key must be 32 bytes long. +func NewSSEC(key []byte) (ServerSide, error) { + if len(key) != 32 { + return nil, errors.New("encrypt: SSE-C key must be 256 bit long") + } + sse := ssec{} + copy(sse[:], key) + return sse, nil +} + +// SSE transforms a SSE-C copy encryption into a SSE-C encryption. +// It is the inverse of SSECopy(...). +// +// If the provided sse is no SSE-C copy encryption SSE returns +// sse unmodified. +func SSE(sse ServerSide) ServerSide { + if sse == nil || sse.Type() != SSEC { + return sse + } + if sse, ok := sse.(ssecCopy); ok { + return ssec(sse) + } + return sse +} + +// SSECopy transforms a SSE-C encryption into a SSE-C copy +// encryption. This is required for SSE-C key rotation or a SSE-C +// copy where the source and the destination should be encrypted. +// +// If the provided sse is no SSE-C encryption SSECopy returns +// sse unmodified. +func SSECopy(sse ServerSide) ServerSide { + if sse == nil || sse.Type() != SSEC { + return sse + } + if sse, ok := sse.(ssec); ok { + return ssecCopy(sse) + } + return sse +} + +type ssec [32]byte + +func (s ssec) Type() Type { return SSEC } + +func (s ssec) Marshal(h http.Header) { + keyMD5 := md5.Sum(s[:]) + h.Set(sseCustomerAlgorithm, "AES256") + h.Set(sseCustomerKey, base64.StdEncoding.EncodeToString(s[:])) + h.Set(sseCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:])) +} + +type ssecCopy [32]byte + +func (s ssecCopy) Type() Type { return SSEC } + +func (s ssecCopy) Marshal(h http.Header) { + keyMD5 := md5.Sum(s[:]) + h.Set(sseCopyCustomerAlgorithm, "AES256") + h.Set(sseCopyCustomerKey, base64.StdEncoding.EncodeToString(s[:])) + h.Set(sseCopyCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:])) +} + +type s3 struct{} + +func (s s3) Type() Type { return S3 } + +func (s s3) Marshal(h http.Header) { h.Set(sseGenericHeader, "AES256") } + +type kms struct { + key string + context []byte + hasContext bool +} + +func (s kms) Type() Type { return KMS } + +func (s kms) Marshal(h http.Header) { + h.Set(sseGenericHeader, "aws:kms") + h.Set(sseKmsKeyID, s.key) + if s.hasContext { + h.Set(sseEncryptionContext, base64.StdEncoding.EncodeToString(s.context)) + } +} diff --git a/pkg/policy/bucket-policy.go b/pkg/policy/bucket-policy.go index 9dda99e..79fd801 100644 --- a/pkg/policy/bucket-policy.go +++ b/pkg/policy/bucket-policy.go @@ -18,6 +18,8 @@ package policy import ( + "encoding/json" + "errors" "reflect" "strings" @@ -82,6 +84,33 @@ type User struct { CanonicalUser set.StringSet `json:"CanonicalUser,omitempty"` } +// UnmarshalJSON is a custom json unmarshaler for Principal field, +// the reason is that Principal can take a json struct represented by +// User string but it can also take a string. +func (u *User) UnmarshalJSON(data []byte) error { + // Try to unmarshal data in a struct equal to User, we need it + // to avoid infinite recursive call of this function + type AliasUser User + var au AliasUser + err := json.Unmarshal(data, &au) + if err == nil { + *u = User(au) + return nil + } + // Data type is not known, check if it is a json string + // which contains a star, which is permitted in the spec + var str string + err = json.Unmarshal(data, &str) + if err == nil { + if str != "*" { + return errors.New("unrecognized Principal field") + } + *u = User{AWS: set.CreateStringSet("*")} + return nil + } + return err +} + // Statement - minio policy statement type Statement struct { Actions set.StringSet `json:"Action"` @@ -528,7 +557,6 @@ func GetPolicy(statements []Statement, bucketName string, prefix string) BucketP } else { matchedObjResources = s.Resources.FuncMatch(resourceMatch, objectResource) } - if !matchedObjResources.IsEmpty() { readOnly, writeOnly := getObjectPolicy(s) for resource := range matchedObjResources { @@ -542,7 +570,8 @@ func GetPolicy(statements []Statement, bucketName string, prefix string) BucketP matchedResource = resource } } - } else if s.Resources.Contains(bucketResource) { + } + if s.Resources.Contains(bucketResource) { commonFound, readOnly, writeOnly := getBucketPolicy(s, prefix) bucketCommonFound = bucketCommonFound || commonFound bucketReadOnly = bucketReadOnly || readOnly @@ -564,18 +593,19 @@ func GetPolicy(statements []Statement, bucketName string, prefix string) BucketP return policy } -// GetPolicies - returns a map of policies rules of given bucket name, prefix in given statements. -func GetPolicies(statements []Statement, bucketName string) map[string]BucketPolicy { +// GetPolicies - returns a map of policies of given bucket name, prefix in given statements. +func GetPolicies(statements []Statement, bucketName, prefix string) map[string]BucketPolicy { policyRules := map[string]BucketPolicy{} objResources := set.NewStringSet() // Search all resources related to objects policy for _, s := range statements { for r := range s.Resources { - if strings.HasPrefix(r, awsResourcePrefix+bucketName+"/") { + if strings.HasPrefix(r, awsResourcePrefix+bucketName+"/"+prefix) { objResources.Add(r) } } } + // Pretend that policy resource as an actual object and fetch its policy for r := range objResources { // Put trailing * if exists in asterisk @@ -584,7 +614,10 @@ func GetPolicies(statements []Statement, bucketName string) map[string]BucketPol r = r[:len(r)-1] asterisk = "*" } - objectPath := r[len(awsResourcePrefix+bucketName)+1:] + var objectPath string + if len(r) >= len(awsResourcePrefix+bucketName)+1 { + objectPath = r[len(awsResourcePrefix+bucketName)+1:] + } p := GetPolicy(statements, bucketName, objectPath) policyRules[bucketName+"/"+objectPath+asterisk] = p } diff --git a/pkg/policy/bucket-policy_test.go b/pkg/policy/bucket-policy_test.go index 1e5196f..1a71d87 100644 --- a/pkg/policy/bucket-policy_test.go +++ b/pkg/policy/bucket-policy_test.go @@ -26,6 +26,205 @@ import ( "github.com/minio/minio-go/pkg/set" ) +// TestUnmarshalBucketPolicy tests unmarsheling various examples +// of bucket policies, to verify the correctness of BucketAccessPolicy +// struct defined in this package. +func TestUnmarshalBucketPolicy(t *testing.T) { + var testCases = []struct { + policyData string + shouldSucceed bool + }{ + // Test 1 + {policyData: `{ + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"AddCannedAcl", + "Effect":"Allow", + "Principal": {"AWS": ["arn:aws:iam::111122223333:root","arn:aws:iam::444455556666:root"]}, + "Action":["s3:PutObject","s3:PutObjectAcl"], + "Resource":["arn:aws:s3:::examplebucket/*"], + "Condition":{"StringEquals":{"s3:x-amz-acl":["public-read"]}} + } + ] +}`, shouldSucceed: true}, + // Test 2 + {policyData: `{ + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"AddPerm", + "Effect":"Allow", + "Principal": "*", + "Action":["s3:GetObject"], + "Resource":["arn:aws:s3:::examplebucket/*"] + } + ] +}`, shouldSucceed: true}, + // Test 3 + {policyData: `{ + "Version": "2012-10-17", + "Id": "S3PolicyId1", + "Statement": [ + { + "Sid": "IPAllow", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:*", + "Resource": "arn:aws:s3:::examplebucket/*", + "Condition": { + "IpAddress": {"aws:SourceIp": "54.240.143.0/24"}, + "NotIpAddress": {"aws:SourceIp": "54.240.143.188/32"} + } + } + ] +}`, shouldSucceed: true}, + // Test 4 + {policyData: `{ + "Id":"PolicyId2", + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"AllowIPmix", + "Effect":"Allow", + "Principal":"*", + "Action":"s3:*", + "Resource":"arn:aws:s3:::examplebucket/*", + "Condition": { + "IpAddress": { + "aws:SourceIp": [ + "54.240.143.0/24", + "2001:DB8:1234:5678::/64" + ] + }, + "NotIpAddress": { + "aws:SourceIp": [ + "54.240.143.128/30", + "2001:DB8:1234:5678:ABCD::/80" + ] + } + } + } + ] +}`, shouldSucceed: true}, + // Test 5 + {policyData: `{ + "Version":"2012-10-17", + "Id":"http referer policy example", + "Statement":[ + { + "Sid":"Allow get requests originating from www.example.com and example.com.", + "Effect":"Allow", + "Principal":"*", + "Action":"s3:GetObject", + "Resource":"arn:aws:s3:::examplebucket/*", + "Condition":{ + "StringLike":{"aws:Referer":["http://www.example.com/*","http://example.com/*"]} + } + } + ] +}`, shouldSucceed: true}, + // Test 6 + {policyData: `{ + "Version": "2012-10-17", + "Id": "http referer policy example", + "Statement": [ + { + "Sid": "Allow get requests referred by www.example.com and example.com.", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::examplebucket/*", + "Condition": { + "StringLike": {"aws:Referer": ["http://www.example.com/*","http://example.com/*"]} + } + }, + { + "Sid": "Explicit deny to ensure requests are allowed only from specific referer.", + "Effect": "Deny", + "Principal": "*", + "Action": "s3:*", + "Resource": "arn:aws:s3:::examplebucket/*", + "Condition": { + "StringNotLike": {"aws:Referer": ["http://www.example.com/*","http://example.com/*"]} + } + } + ] +}`, shouldSucceed: true}, + + // Test 7 + {policyData: `{ + "Version":"2012-10-17", + "Id":"PolicyForCloudFrontPrivateContent", + "Statement":[ + { + "Sid":" Grant a CloudFront Origin Identity access to support private content", + "Effect":"Allow", + "Principal":{"CanonicalUser":"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be"}, + "Action":"s3:GetObject", + "Resource":"arn:aws:s3:::example-bucket/*" + } + ] +}`, shouldSucceed: true}, + // Test 8 + {policyData: `{ + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"111", + "Effect":"Allow", + "Principal":{"AWS":"1111111111"}, + "Action":"s3:PutObject", + "Resource":"arn:aws:s3:::examplebucket/*" + }, + { + "Sid":"112", + "Effect":"Deny", + "Principal":{"AWS":"1111111111" }, + "Action":"s3:PutObject", + "Resource":"arn:aws:s3:::examplebucket/*", + "Condition": { + "StringNotEquals": {"s3:x-amz-grant-full-control":["emailAddress=xyz@amazon.com"]} + } + } + ] +}`, shouldSucceed: true}, + // Test 9 + {policyData: `{ + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"InventoryAndAnalyticsExamplePolicy", + "Effect":"Allow", + "Principal": {"Service": "s3.amazonaws.com"}, + "Action":["s3:PutObject"], + "Resource":["arn:aws:s3:::destination-bucket/*"], + "Condition": { + "ArnLike": { + "aws:SourceArn": "arn:aws:s3:::source-bucket" + }, + "StringEquals": { + "aws:SourceAccount": "1234567890", + "s3:x-amz-acl": "bucket-owner-full-control" + } + } + } + ] +}`, shouldSucceed: true}, + } + + for i, testCase := range testCases { + var policy BucketAccessPolicy + err := json.Unmarshal([]byte(testCase.policyData), &policy) + if testCase.shouldSucceed && err != nil { + t.Fatalf("Test %d: expected to succeed but it has an error: %v", i+1, err) + } + if !testCase.shouldSucceed && err == nil { + t.Fatalf("Test %d: expected to fail but succeeded", i+1) + } + } +} + // isValidStatement() is called and the result is validated. func TestIsValidStatement(t *testing.T) { testCases := []struct { @@ -1393,6 +1592,7 @@ func TestListBucketPolicies(t *testing.T) { downloadUploadCondKeyMap.Add("s3:prefix", set.CreateStringSet("both")) downloadUploadCondMap.Add("StringEquals", downloadUploadCondKeyMap) + commonSetActions := commonBucketActions.Union(readOnlyBucketActions) testCases := []struct { statements []Statement bucketName string @@ -1431,6 +1631,13 @@ func TestListBucketPolicies(t *testing.T) { Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/download*"), }}, "mybucket", "", map[string]BucketPolicy{"mybucket/download*": BucketPolicyReadOnly}}, + {[]Statement{ + { + Actions: commonSetActions.Union(readOnlyObjectActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket", "arn:aws:s3:::mybucket/*"), + }}, "mybucket", "", map[string]BucketPolicy{"mybucket/*": BucketPolicyReadOnly}}, // Write Only {[]Statement{ { @@ -1469,7 +1676,7 @@ func TestListBucketPolicies(t *testing.T) { } for _, testCase := range testCases { - policyRules := GetPolicies(testCase.statements, testCase.bucketName) + policyRules := GetPolicies(testCase.statements, testCase.bucketName, "") if !reflect.DeepEqual(testCase.expectedResult, policyRules) { t.Fatalf("%+v:\n expected: %+v, got: %+v", testCase, testCase.expectedResult, policyRules) } diff --git a/pkg/s3signer/request-signature-streaming_test.go b/pkg/s3signer/request-signature-streaming_test.go index 535adb3..297ab97 100644 --- a/pkg/s3signer/request-signature-streaming_test.go +++ b/pkg/s3signer/request-signature-streaming_test.go @@ -33,7 +33,7 @@ func TestGetSeedSignature(t *testing.T) { req := NewRequest("PUT", "/examplebucket/chunkObject.txt", body) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") - req.URL.Host = "s3.amazonaws.com" + req.Host = "s3.amazonaws.com" reqTime, err := time.Parse("20060102T150405Z", "20130524T000000Z") if err != nil { @@ -69,6 +69,7 @@ func TestSetStreamingAuthorization(t *testing.T) { req := NewRequest("PUT", "/examplebucket/chunkObject.txt", nil) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + req.Host = "" req.URL.Host = "s3.amazonaws.com" dataLen := int64(65 * 1024) @@ -93,6 +94,7 @@ func TestStreamingReader(t *testing.T) { req := NewRequest("PUT", "/examplebucket/chunkObject.txt", nil) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") req.ContentLength = 65 * 1024 + req.Host = "" req.URL.Host = "s3.amazonaws.com" baseReader := ioutil.NopCloser(bytes.NewReader(bytes.Repeat([]byte("a"), 65*1024))) diff --git a/pkg/s3signer/request-signature-v2.go b/pkg/s3signer/request-signature-v2.go index 620af1c..b407093 100644 --- a/pkg/s3signer/request-signature-v2.go +++ b/pkg/s3signer/request-signature-v2.go @@ -25,7 +25,6 @@ import ( "fmt" "net/http" "net/url" - "path/filepath" "sort" "strconv" "strings" @@ -40,28 +39,25 @@ const ( ) // Encode input URL path to URL encoded path. -func encodeURL2Path(u *url.URL) (path string) { - // Encode URL path. - if isS3, _ := filepath.Match("*.s3*.amazonaws.com", u.Host); isS3 { - bucketName := u.Host[:strings.LastIndex(u.Host, ".s3")] - path = "/" + bucketName - path += u.Path - path = s3utils.EncodePath(path) - return - } - if strings.HasSuffix(u.Host, ".storage.googleapis.com") { - path = "/" + strings.TrimSuffix(u.Host, ".storage.googleapis.com") - path += u.Path - path = s3utils.EncodePath(path) - return +func encodeURL2Path(req *http.Request, virtualHost bool) (path string) { + if virtualHost { + reqHost := getHostAddr(req) + dotPos := strings.Index(reqHost, ".") + if dotPos > -1 { + bucketName := reqHost[:dotPos] + path = "/" + bucketName + path += req.URL.Path + path = s3utils.EncodePath(path) + return + } } - path = s3utils.EncodePath(u.Path) + path = s3utils.EncodePath(req.URL.Path) return } // PreSignV2 - presign the request in following style. // https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}. -func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires int64) *http.Request { +func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires int64, virtualHost bool) *http.Request { // Presign is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { return &req @@ -77,7 +73,7 @@ func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires in } // Get presigned string to sign. - stringToSign := preStringToSignV2(req) + stringToSign := preStringToSignV2(req, virtualHost) hm := hmac.New(sha1.New, []byte(secretAccessKey)) hm.Write([]byte(stringToSign)) @@ -86,7 +82,7 @@ func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires in query := req.URL.Query() // Handle specially for Google Cloud Storage. - if strings.Contains(req.URL.Host, ".storage.googleapis.com") { + if strings.Contains(getHostAddr(&req), ".storage.googleapis.com") { query.Set("GoogleAccessId", accessKeyID) } else { query.Set("AWSAccessKeyId", accessKeyID) @@ -131,7 +127,7 @@ func PostPresignSignatureV2(policyBase64, secretAccessKey string) string { // CanonicalizedProtocolHeaders = // SignV2 sign the request before Do() (AWS Signature Version 2). -func SignV2(req http.Request, accessKeyID, secretAccessKey string) *http.Request { +func SignV2(req http.Request, accessKeyID, secretAccessKey string, virtualHost bool) *http.Request { // Signature calculation is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { return &req @@ -146,7 +142,7 @@ func SignV2(req http.Request, accessKeyID, secretAccessKey string) *http.Request } // Calculate HMAC for secretAccessKey. - stringToSign := stringToSignV2(req) + stringToSign := stringToSignV2(req, virtualHost) hm := hmac.New(sha1.New, []byte(secretAccessKey)) hm.Write([]byte(stringToSign)) @@ -171,14 +167,14 @@ func SignV2(req http.Request, accessKeyID, secretAccessKey string) *http.Request // Expires + "\n" + // CanonicalizedProtocolHeaders + // CanonicalizedResource; -func preStringToSignV2(req http.Request) string { +func preStringToSignV2(req http.Request, virtualHost bool) string { buf := new(bytes.Buffer) // Write standard headers. writePreSignV2Headers(buf, req) // Write canonicalized protocol headers if any. writeCanonicalizedHeaders(buf, req) // Write canonicalized Query resources if any. - writeCanonicalizedResource(buf, req) + writeCanonicalizedResource(buf, req, virtualHost) return buf.String() } @@ -198,14 +194,14 @@ func writePreSignV2Headers(buf *bytes.Buffer, req http.Request) { // Date + "\n" + // CanonicalizedProtocolHeaders + // CanonicalizedResource; -func stringToSignV2(req http.Request) string { +func stringToSignV2(req http.Request, virtualHost bool) string { buf := new(bytes.Buffer) // Write standard headers. writeSignV2Headers(buf, req) // Write canonicalized protocol headers if any. writeCanonicalizedHeaders(buf, req) // Write canonicalized Query resources if any. - writeCanonicalizedResource(buf, req) + writeCanonicalizedResource(buf, req, virtualHost) return buf.String() } @@ -287,11 +283,11 @@ var resourceList = []string{ // CanonicalizedResource = [ "/" + Bucket ] + // + // [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; -func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request) { +func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request, virtualHost bool) { // Save request URL. requestURL := req.URL // Get encoded URL path. - buf.WriteString(encodeURL2Path(requestURL)) + buf.WriteString(encodeURL2Path(&req, virtualHost)) if requestURL.RawQuery != "" { var n int vals, _ := url.ParseQuery(requestURL.RawQuery) diff --git a/pkg/s3signer/request-signature-v4.go b/pkg/s3signer/request-signature-v4.go index d5721ac..daf02fe 100644 --- a/pkg/s3signer/request-signature-v4.go +++ b/pkg/s3signer/request-signature-v4.go @@ -144,7 +144,7 @@ func getCanonicalHeaders(req http.Request, ignoredHeaders map[string]bool) strin buf.WriteByte(':') switch { case k == "host": - buf.WriteString(req.URL.Host) + buf.WriteString(getHostAddr(&req)) fallthrough default: for idx, v := range vals[k] { diff --git a/pkg/s3signer/request-signature-v4_test.go b/pkg/s3signer/request-signature-v4_test.go new file mode 100644 index 0000000..a109a4f --- /dev/null +++ b/pkg/s3signer/request-signature-v4_test.go @@ -0,0 +1,50 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "io" + "net/http" + "strings" + "testing" +) + +func TestRequestHost(t *testing.T) { + req, _ := buildRequest("dynamodb", "us-east-1", "{}") + req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a" + req.Host = "myhost" + canonicalHeaders := getCanonicalHeaders(*req, v4IgnoredHeaders) + + if !strings.Contains(canonicalHeaders, "host:"+req.Host) { + t.Errorf("canonical host header invalid") + } +} + +func buildRequest(serviceName, region, body string) (*http.Request, io.ReadSeeker) { + endpoint := "https://" + serviceName + "." + region + ".amazonaws.com" + reader := strings.NewReader(body) + req, _ := http.NewRequest("POST", endpoint, reader) + req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()" + req.Header.Add("X-Amz-Target", "prefix.Operation") + req.Header.Add("Content-Type", "application/x-amz-json-1.0") + req.Header.Add("Content-Length", string(len(body))) + req.Header.Add("X-Amz-Meta-Other-Header", "some-value=!@#$%^&* (+)") + req.Header.Add("X-Amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)") + req.Header.Add("X-amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)") + return req, reader +} diff --git a/pkg/s3signer/request-signature_test.go b/pkg/s3signer/request-signature_test.go index d53483e..75115d1 100644 --- a/pkg/s3signer/request-signature_test.go +++ b/pkg/s3signer/request-signature_test.go @@ -24,7 +24,7 @@ import ( ) // Tests signature calculation. -func TestSignatureCalculation(t *testing.T) { +func TestSignatureCalculationV4(t *testing.T) { req, err := http.NewRequest("GET", "https://s3.amazonaws.com", nil) if err != nil { t.Fatal("Error:", err) @@ -39,16 +39,6 @@ func TestSignatureCalculation(t *testing.T) { t.Fatal("Error: anonymous credentials should not have Signature query resource.") } - req = SignV2(*req, "", "") - if req.Header.Get("Authorization") != "" { - t.Fatal("Error: anonymous credentials should not have Authorization header.") - } - - req = PreSignV2(*req, "", "", 0) - if strings.Contains(req.URL.RawQuery, "Signature") { - t.Fatal("Error: anonymous credentials should not have Signature query resource.") - } - req = SignV4(*req, "ACCESS-KEY", "SECRET-KEY", "", "us-east-1") if req.Header.Get("Authorization") == "" { t.Fatal("Error: normal credentials should have Authorization header.") @@ -58,14 +48,42 @@ func TestSignatureCalculation(t *testing.T) { if !strings.Contains(req.URL.RawQuery, "X-Amz-Signature") { t.Fatal("Error: normal credentials should have Signature query resource.") } +} - req = SignV2(*req, "ACCESS-KEY", "SECRET-KEY") - if req.Header.Get("Authorization") == "" { - t.Fatal("Error: normal credentials should have Authorization header.") +func TestSignatureCalculationV2(t *testing.T) { + + var testCases = []struct { + endpointURL string + virtualHost bool + }{ + {endpointURL: "https://s3.amazonaws.com/", virtualHost: false}, + {endpointURL: "https://testbucket.s3.amazonaws.com/", virtualHost: true}, } - req = PreSignV2(*req, "ACCESS-KEY", "SECRET-KEY", 0) - if !strings.Contains(req.URL.RawQuery, "Signature") { - t.Fatal("Error: normal credentials should not have Signature query resource.") + for i, testCase := range testCases { + req, err := http.NewRequest("GET", testCase.endpointURL, nil) + if err != nil { + t.Fatalf("Test %d, Error: %v", i+1, err) + } + + req = SignV2(*req, "", "", testCase.virtualHost) + if req.Header.Get("Authorization") != "" { + t.Fatalf("Test %d, Error: anonymous credentials should not have Authorization header.", i+1) + } + + req = PreSignV2(*req, "", "", 0, testCase.virtualHost) + if strings.Contains(req.URL.RawQuery, "Signature") { + t.Fatalf("Test %d, Error: anonymous credentials should not have Signature query resource.", i+1) + } + + req = SignV2(*req, "ACCESS-KEY", "SECRET-KEY", testCase.virtualHost) + if req.Header.Get("Authorization") == "" { + t.Fatalf("Test %d, Error: normal credentials should have Authorization header.", i+1) + } + + req = PreSignV2(*req, "ACCESS-KEY", "SECRET-KEY", 0, testCase.virtualHost) + if !strings.Contains(req.URL.RawQuery, "Signature") { + t.Fatalf("Test %d, Error: normal credentials should not have Signature query resource.", i+1) + } } } diff --git a/pkg/s3signer/utils.go b/pkg/s3signer/utils.go index 2924363..33b1752 100644 --- a/pkg/s3signer/utils.go +++ b/pkg/s3signer/utils.go @@ -20,6 +20,7 @@ package s3signer import ( "crypto/hmac" "crypto/sha256" + "net/http" ) // unsignedPayload - value to be set to X-Amz-Content-Sha256 header when @@ -38,3 +39,11 @@ func sumHMAC(key []byte, data []byte) []byte { hash.Write(data) return hash.Sum(nil) } + +// getHostAddr returns host header if available, otherwise returns host from URL +func getHostAddr(req *http.Request) string { + if req.Host != "" { + return req.Host + } + return req.URL.Host +} diff --git a/pkg/s3signer/utils_test.go b/pkg/s3signer/utils_test.go index 22a2d65..e7fc7b3 100644 --- a/pkg/s3signer/utils_test.go +++ b/pkg/s3signer/utils_test.go @@ -19,6 +19,7 @@ package s3signer import ( "fmt" + "net/http" "net/url" "testing" ) @@ -26,48 +27,59 @@ import ( // Tests url encoding. func TestEncodeURL2Path(t *testing.T) { type urlStrings struct { + virtualHost bool bucketName string objName string encodedObjName string } - bucketName := "bucketName" want := []urlStrings{ { + virtualHost: true, bucketName: "bucketName", objName: "本語", encodedObjName: "%E6%9C%AC%E8%AA%9E", }, { + virtualHost: true, bucketName: "bucketName", objName: "本語.1", encodedObjName: "%E6%9C%AC%E8%AA%9E.1", }, { + virtualHost: true, objName: ">123>3123123", bucketName: "bucketName", encodedObjName: "%3E123%3E3123123", }, { + virtualHost: true, bucketName: "bucketName", objName: "test 1 2.txt", encodedObjName: "test%201%202.txt", }, { + virtualHost: false, bucketName: "test.bucketName", objName: "test++ 1.txt", encodedObjName: "test%2B%2B%201.txt", }, } - for _, o := range want { - u, err := url.Parse(fmt.Sprintf("https://%s.s3.amazonaws.com/%s", bucketName, o.objName)) + for i, o := range want { + var hostURL string + if o.virtualHost { + hostURL = fmt.Sprintf("https://%s.s3.amazonaws.com/%s", o.bucketName, o.objName) + } else { + hostURL = fmt.Sprintf("https://s3.amazonaws.com/%s/%s", o.bucketName, o.objName) + } + u, err := url.Parse(hostURL) if err != nil { - t.Fatal("Error:", err) + t.Fatalf("Test %d, Error: %v", i+1, err) } - urlPath := "/" + bucketName + "/" + o.encodedObjName - if urlPath != encodeURL2Path(u) { - t.Fatal("Error") + expectedPath := "/" + o.bucketName + "/" + o.encodedObjName + if foundPath := encodeURL2Path(&http.Request{URL: u}, o.virtualHost); foundPath != expectedPath { + t.Fatalf("Test %d, Error: expected = `%v`, found = `%v`", i+1, expectedPath, foundPath) } } diff --git a/pkg/s3utils/utils.go b/pkg/s3utils/utils.go index 258390f..adceb7f 100644 --- a/pkg/s3utils/utils.go +++ b/pkg/s3utils/utils.go @@ -81,18 +81,56 @@ func IsVirtualHostSupported(endpointURL url.URL, bucketName string) bool { return IsAmazonEndpoint(endpointURL) || IsGoogleEndpoint(endpointURL) } -// AmazonS3Host - regular expression used to determine if an arg is s3 host. -var AmazonS3Host = regexp.MustCompile("^s3[.-]?(.*?)\\.amazonaws\\.com$") +// Refer for region styles - https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region -// IsAmazonEndpoint - Match if it is exactly Amazon S3 endpoint. -func IsAmazonEndpoint(endpointURL url.URL) bool { - if IsAmazonChinaEndpoint(endpointURL) { - return true +// amazonS3HostHyphen - regular expression used to determine if an arg is s3 host in hyphenated style. +var amazonS3HostHyphen = regexp.MustCompile(`^s3-(.*?)\.amazonaws\.com$`) + +// amazonS3HostDualStack - regular expression used to determine if an arg is s3 host dualstack. +var amazonS3HostDualStack = regexp.MustCompile(`^s3\.dualstack\.(.*?)\.amazonaws\.com$`) + +// amazonS3HostDot - regular expression used to determine if an arg is s3 host in . style. +var amazonS3HostDot = regexp.MustCompile(`^s3\.(.*?)\.amazonaws\.com$`) + +// amazonS3ChinaHost - regular expression used to determine if the arg is s3 china host. +var amazonS3ChinaHost = regexp.MustCompile(`^s3\.(cn.*?)\.amazonaws\.com\.cn$`) + +// GetRegionFromURL - returns a region from url host. +func GetRegionFromURL(endpointURL url.URL) string { + if endpointURL == sentinelURL { + return "" + } + if endpointURL.Host == "s3-external-1.amazonaws.com" { + return "" } if IsAmazonGovCloudEndpoint(endpointURL) { + return "us-gov-west-1" + } + parts := amazonS3HostDualStack.FindStringSubmatch(endpointURL.Host) + if len(parts) > 1 { + return parts[1] + } + parts = amazonS3HostHyphen.FindStringSubmatch(endpointURL.Host) + if len(parts) > 1 { + return parts[1] + } + parts = amazonS3ChinaHost.FindStringSubmatch(endpointURL.Host) + if len(parts) > 1 { + return parts[1] + } + parts = amazonS3HostDot.FindStringSubmatch(endpointURL.Host) + if len(parts) > 1 { + return parts[1] + } + return "" +} + +// IsAmazonEndpoint - Match if it is exactly Amazon S3 endpoint. +func IsAmazonEndpoint(endpointURL url.URL) bool { + if endpointURL.Host == "s3-external-1.amazonaws.com" || endpointURL.Host == "s3.amazonaws.com" { return true } - return AmazonS3Host.MatchString(endpointURL.Host) + return GetRegionFromURL(endpointURL) != "" } // IsAmazonGovCloudEndpoint - Match if it is exactly Amazon S3 GovCloud endpoint. @@ -105,24 +143,40 @@ func IsAmazonGovCloudEndpoint(endpointURL url.URL) bool { } // IsAmazonFIPSGovCloudEndpoint - Match if it is exactly Amazon S3 FIPS GovCloud endpoint. +// See https://aws.amazon.com/compliance/fips. func IsAmazonFIPSGovCloudEndpoint(endpointURL url.URL) bool { if endpointURL == sentinelURL { return false } - return endpointURL.Host == "s3-fips-us-gov-west-1.amazonaws.com" + return endpointURL.Host == "s3-fips-us-gov-west-1.amazonaws.com" || + endpointURL.Host == "s3-fips.dualstack.us-gov-west-1.amazonaws.com" } -// IsAmazonChinaEndpoint - Match if it is exactly Amazon S3 China endpoint. -// Customers who wish to use the new Beijing Region are required -// to sign up for a separate set of account credentials unique to -// the China (Beijing) Region. Customers with existing AWS credentials -// will not be able to access resources in the new Region, and vice versa. -// For more info https://aws.amazon.com/about-aws/whats-new/2013/12/18/announcing-the-aws-china-beijing-region/ -func IsAmazonChinaEndpoint(endpointURL url.URL) bool { +// IsAmazonFIPSUSEastWestEndpoint - Match if it is exactly Amazon S3 FIPS US East/West endpoint. +// See https://aws.amazon.com/compliance/fips. +func IsAmazonFIPSUSEastWestEndpoint(endpointURL url.URL) bool { if endpointURL == sentinelURL { return false } - return endpointURL.Host == "s3.cn-north-1.amazonaws.com.cn" + switch endpointURL.Host { + case "s3-fips.us-east-2.amazonaws.com": + case "s3-fips.dualstack.us-west-1.amazonaws.com": + case "s3-fips.dualstack.us-west-2.amazonaws.com": + case "s3-fips.dualstack.us-east-2.amazonaws.com": + case "s3-fips.dualstack.us-east-1.amazonaws.com": + case "s3-fips.us-west-1.amazonaws.com": + case "s3-fips.us-west-2.amazonaws.com": + case "s3-fips.us-east-1.amazonaws.com": + default: + return false + } + return true +} + +// IsAmazonFIPSEndpoint - Match if it is exactly Amazon S3 FIPS endpoint. +// See https://aws.amazon.com/compliance/fips. +func IsAmazonFIPSEndpoint(endpointURL url.URL) bool { + return IsAmazonFIPSUSEastWestEndpoint(endpointURL) || IsAmazonFIPSGovCloudEndpoint(endpointURL) } // IsGoogleEndpoint - Match if it is exactly Google cloud storage endpoint. diff --git a/pkg/s3utils/utils_test.go b/pkg/s3utils/utils_test.go index f19e688..55eaaea 100644 --- a/pkg/s3utils/utils_test.go +++ b/pkg/s3utils/utils_test.go @@ -23,6 +23,66 @@ import ( "testing" ) +// Tests get region from host URL. +func TestGetRegionFromURL(t *testing.T) { + testCases := []struct { + u url.URL + expectedRegion string + }{ + { + u: url.URL{Host: "storage.googleapis.com"}, + expectedRegion: "", + }, + { + u: url.URL{Host: "s3.cn-north-1.amazonaws.com.cn"}, + expectedRegion: "cn-north-1", + }, + { + u: url.URL{Host: "s3.cn-northwest-1.amazonaws.com.cn"}, + expectedRegion: "cn-northwest-1", + }, + { + u: url.URL{Host: "s3-fips-us-gov-west-1.amazonaws.com"}, + expectedRegion: "us-gov-west-1", + }, + { + u: url.URL{Host: "s3-us-gov-west-1.amazonaws.com"}, + expectedRegion: "us-gov-west-1", + }, + { + u: url.URL{Host: "192.168.1.1"}, + expectedRegion: "", + }, + { + u: url.URL{Host: "s3-eu-west-1.amazonaws.com"}, + expectedRegion: "eu-west-1", + }, + { + u: url.URL{Host: "s3.eu-west-1.amazonaws.com"}, + expectedRegion: "eu-west-1", + }, + { + u: url.URL{Host: "s3.dualstack.eu-west-1.amazonaws.com"}, + expectedRegion: "eu-west-1", + }, + { + u: url.URL{Host: "s3.amazonaws.com"}, + expectedRegion: "", + }, + { + u: url.URL{Host: "s3-external-1.amazonaws.com"}, + expectedRegion: "", + }, + } + + for i, testCase := range testCases { + region := GetRegionFromURL(testCase.u) + if testCase.expectedRegion != region { + t.Errorf("Test %d: Expected region %s, got %s", i+1, testCase.expectedRegion, region) + } + } +} + // Tests for 'isValidDomain(host string) bool'. func TestIsValidDomain(t *testing.T) { testCases := []struct { @@ -33,6 +93,7 @@ func TestIsValidDomain(t *testing.T) { }{ {"s3.amazonaws.com", true}, {"s3.cn-north-1.amazonaws.com.cn", true}, + {"s3.cn-northwest-1.amazonaws.com.cn", true}, {"s3.amazonaws.com_", false}, {"%$$$", false}, {"s3.amz.test.com", true}, @@ -120,9 +181,17 @@ func TestIsAmazonEndpoint(t *testing.T) { {"https://amazons3.amazonaws.com", false}, {"-192.168.1.1", false}, {"260.192.1.1", false}, + {"https://s3-.amazonaws.com", false}, + {"https://s3..amazonaws.com", false}, + {"https://s3.dualstack.us-west-1.amazonaws.com.cn", false}, + {"https://s3..us-west-1.amazonaws.com.cn", false}, // valid inputs. {"https://s3.amazonaws.com", true}, + {"https://s3-external-1.amazonaws.com", true}, {"https://s3.cn-north-1.amazonaws.com.cn", true}, + {"https://s3-us-west-1.amazonaws.com", true}, + {"https://s3.us-west-1.amazonaws.com", true}, + {"https://s3.dualstack.us-west-1.amazonaws.com", true}, } for i, testCase := range testCases { @@ -138,41 +207,6 @@ func TestIsAmazonEndpoint(t *testing.T) { } -// Tests validate Amazon S3 China endpoint validator. -func TestIsAmazonChinaEndpoint(t *testing.T) { - testCases := []struct { - url string - // Expected result. - result bool - }{ - {"https://192.168.1.1", false}, - {"192.168.1.1", false}, - {"http://storage.googleapis.com", false}, - {"https://storage.googleapis.com", false}, - {"storage.googleapis.com", false}, - {"s3.amazonaws.com", false}, - {"https://amazons3.amazonaws.com", false}, - {"-192.168.1.1", false}, - {"260.192.1.1", false}, - // s3.amazonaws.com is not a valid Amazon S3 China end point. - {"https://s3.amazonaws.com", false}, - // valid input. - {"https://s3.cn-north-1.amazonaws.com.cn", true}, - } - - for i, testCase := range testCases { - u, err := url.Parse(testCase.url) - if err != nil { - t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err) - } - result := IsAmazonChinaEndpoint(*u) - if testCase.result != result { - t.Errorf("Test %d: Expected isAmazonEndpoint to be '%v' for input \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.url, result) - } - } - -} - // Tests validate Google Cloud end point validator. func TestIsGoogleEndpoint(t *testing.T) { testCases := []struct { diff --git a/post-policy.go b/post-policy.go index b3ae705..c285fde 100644 --- a/post-policy.go +++ b/post-policy.go @@ -206,6 +206,28 @@ func (p *PostPolicy) SetUserMetadata(key string, value string) error { return nil } +// SetUserData - Set user data as a key/value couple. +// Can be retrieved through a HEAD request or an event. +func (p *PostPolicy) SetUserData(key string, value string) error { + if key == "" { + return ErrInvalidArgument("Key is empty") + } + if value == "" { + return ErrInvalidArgument("Value is empty") + } + headerName := fmt.Sprintf("x-amz-%s", key) + policyCond := policyCondition{ + matchType: "eq", + condition: fmt.Sprintf("$%s", headerName), + value: value, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData[headerName] = value + return nil +} + // addNewPolicy - internal helper to validate adding new policies. func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error { if policyCond.matchType == "" || policyCond.condition == "" || policyCond.value == "" { diff --git a/retry.go b/retry.go index 2c8ceda..2a76707 100644 --- a/retry.go +++ b/retry.go @@ -26,7 +26,7 @@ import ( ) // MaxRetry is the maximum number of retries before stopping. -var MaxRetry = 5 +var MaxRetry = 10 // MaxJitter will randomize over the full exponential backoff time const MaxJitter = 1.0 @@ -85,33 +85,32 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio return attemptCh } -// isNetErrorRetryable - is network error retryable. -func isNetErrorRetryable(err error) bool { +// isHTTPReqErrorRetryable - is http requests error retryable, such +// as i/o timeout, connection broken etc.. +func isHTTPReqErrorRetryable(err error) bool { if err == nil { return false } - switch err.(type) { - case net.Error: - switch err.(type) { + switch e := err.(type) { + case *url.Error: + switch e.Err.(type) { case *net.DNSError, *net.OpError, net.UnknownNetworkError: return true - case *url.Error: - // For a URL error, where it replies back "connection closed" - // retry again. - if strings.Contains(err.Error(), "Connection closed by foreign host") { - return true - } - default: - if strings.Contains(err.Error(), "net/http: TLS handshake timeout") { - // If error is - tlsHandshakeTimeoutError, retry. - return true - } else if strings.Contains(err.Error(), "i/o timeout") { - // If error is - tcp timeoutError, retry. - return true - } else if strings.Contains(err.Error(), "connection timed out") { - // If err is a net.Dial timeout, retry. - return true - } + } + if strings.Contains(err.Error(), "Connection closed by foreign host") { + return true + } else if strings.Contains(err.Error(), "net/http: TLS handshake timeout") { + // If error is - tlsHandshakeTimeoutError, retry. + return true + } else if strings.Contains(err.Error(), "i/o timeout") { + // If error is - tcp timeoutError, retry. + return true + } else if strings.Contains(err.Error(), "connection timed out") { + // If err is a net.Dial timeout, retry. + return true + } else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") { + // If error is transport connection broken, retry. + return true } } return false @@ -128,6 +127,7 @@ var retryableS3Codes = map[string]struct{}{ "InternalError": {}, "ExpiredToken": {}, "ExpiredTokenException": {}, + "SlowDown": {}, // Add more AWS S3 codes here. } diff --git a/s3-endpoints.go b/s3-endpoints.go index 2a86eaa..0589295 100644 --- a/s3-endpoints.go +++ b/s3-endpoints.go @@ -18,15 +18,15 @@ package minio // awsS3EndpointMap Amazon S3 endpoint map. -// "cn-north-1" adds support for AWS China. var awsS3EndpointMap = map[string]string{ "us-east-1": "s3.amazonaws.com", "us-east-2": "s3-us-east-2.amazonaws.com", "us-west-2": "s3-us-west-2.amazonaws.com", "us-west-1": "s3-us-west-1.amazonaws.com", - "ca-central-1": "s3.ca-central-1.amazonaws.com", + "ca-central-1": "s3-ca-central-1.amazonaws.com", "eu-west-1": "s3-eu-west-1.amazonaws.com", "eu-west-2": "s3-eu-west-2.amazonaws.com", + "eu-west-3": "s3-eu-west-3.amazonaws.com", "eu-central-1": "s3-eu-central-1.amazonaws.com", "ap-south-1": "s3-ap-south-1.amazonaws.com", "ap-southeast-1": "s3-ap-southeast-1.amazonaws.com", @@ -36,6 +36,7 @@ var awsS3EndpointMap = map[string]string{ "sa-east-1": "s3-sa-east-1.amazonaws.com", "us-gov-west-1": "s3-us-gov-west-1.amazonaws.com", "cn-north-1": "s3.cn-north-1.amazonaws.com.cn", + "cn-northwest-1": "s3.cn-northwest-1.amazonaws.com.cn", } // getS3Endpoint get Amazon S3 endpoint based on the bucket location. diff --git a/s3-error.go b/s3-error.go index f9e8233..3b11776 100644 --- a/s3-error.go +++ b/s3-error.go @@ -34,7 +34,7 @@ var s3ErrorResponseMap = map[string]string{ "MissingContentLength": "You must provide the Content-Length HTTP header.", "MissingContentMD5": "Missing required header for this request: Content-Md5.", "MissingRequestBodyError": "Request body is empty.", - "NoSuchBucket": "The specified bucket does not exist", + "NoSuchBucket": "The specified bucket does not exist.", "NoSuchBucketPolicy": "The bucket policy does not exist", "NoSuchKey": "The specified key does not exist.", "NoSuchUpload": "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.", diff --git a/transport.go b/transport.go index e2dafe1..88700cf 100644 --- a/transport.go +++ b/transport.go @@ -2,7 +2,7 @@ /* * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * Copyright 2017-2018 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,9 +25,10 @@ import ( "time" ) -// This default transport is similar to http.DefaultTransport -// but with additional DisableCompression: -var defaultMinioTransport http.RoundTripper = &http.Transport{ +// DefaultTransport - this default transport is similar to +// http.DefaultTransport but with additional param DisableCompression +// is set to true to avoid decompressing content with 'gzip' encoding. +var DefaultTransport http.RoundTripper = &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, @@ -35,6 +36,7 @@ var defaultMinioTransport http.RoundTripper = &http.Transport{ DualStack: true, }).DialContext, MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, diff --git a/utils.go b/utils.go index a8ff8cf..8483f38 100644 --- a/utils.go +++ b/utils.go @@ -209,14 +209,11 @@ func getDefaultLocation(u url.URL, regionOverride string) (location string) { if regionOverride != "" { return regionOverride } - if s3utils.IsAmazonChinaEndpoint(u) { - return "cn-north-1" + region := s3utils.GetRegionFromURL(u) + if region == "" { + region = "us-east-1" } - if s3utils.IsAmazonGovCloudEndpoint(u) { - return "us-gov-west-1" - } - // Default to location to 'us-east-1'. - return "us-east-1" + return region } var supportedHeaders = []string{ @@ -224,14 +221,15 @@ var supportedHeaders = []string{ "cache-control", "content-encoding", "content-disposition", + "content-language", + "x-amz-website-redirect-location", + "expires", // Add more supported headers here. } -// cseHeaders is list of client side encryption headers -var cseHeaders = []string{ - "X-Amz-Iv", - "X-Amz-Key", - "X-Amz-Matdesc", +// isStorageClassHeader returns true if the header is a supported storage class header +func isStorageClassHeader(headerKey string) bool { + return strings.ToLower(amzStorageClass) == strings.ToLower(headerKey) } // isStandardHeader returns true if header is a supported header and not a custom header @@ -245,19 +243,6 @@ func isStandardHeader(headerKey string) bool { return false } -// isCSEHeader returns true if header is a client side encryption header. -func isCSEHeader(headerKey string) bool { - key := strings.ToLower(headerKey) - for _, h := range cseHeaders { - header := strings.ToLower(h) - if (header == key) || - (("x-amz-meta-" + header) == key) { - return true - } - } - return false -} - // sseHeaders is list of server side encryption headers var sseHeaders = []string{ "x-amz-server-side-encryption", @@ -283,5 +268,5 @@ func isSSEHeader(headerKey string) bool { func isAmzHeader(headerKey string) bool { key := strings.ToLower(headerKey) - return strings.HasPrefix(key, "x-amz-meta-") || key == "x-amz-acl" + return strings.HasPrefix(key, "x-amz-meta-") || strings.HasPrefix(key, "x-amz-grant-") || key == "x-amz-acl" || isSSEHeader(headerKey) } diff --git a/utils_test.go b/utils_test.go index 719ee4b..2e60f77 100644 --- a/utils_test.go +++ b/utils_test.go @@ -82,8 +82,10 @@ func TestGetEndpointURL(t *testing.T) { }{ {"s3.amazonaws.com", true, "https://s3.amazonaws.com", nil, true}, {"s3.cn-north-1.amazonaws.com.cn", true, "https://s3.cn-north-1.amazonaws.com.cn", nil, true}, + {"s3.cn-northwest-1.amazonaws.com.cn", true, "https://s3.cn-northwest-1.amazonaws.com.cn", nil, true}, {"s3.amazonaws.com", false, "http://s3.amazonaws.com", nil, true}, {"s3.cn-north-1.amazonaws.com.cn", false, "http://s3.cn-north-1.amazonaws.com.cn", nil, true}, + {"s3.cn-northwest-1.amazonaws.com.cn", false, "http://s3.cn-northwest-1.amazonaws.com.cn", nil, true}, {"192.168.1.1:9000", false, "http://192.168.1.1:9000", nil, true}, {"192.168.1.1:9000", true, "https://192.168.1.1:9000", nil, true}, {"s3.amazonaws.com:443", true, "https://s3.amazonaws.com:443", nil, true}, @@ -200,7 +202,13 @@ func TestDefaultBucketLocation(t *testing.T) { regionOverride: "", expectedLocation: "cn-north-1", }, - // No region provided, no standard region strings provided as well. - Test 5. + // China region should be honored, region override not provided. - Test 5. + { + endpointURL: url.URL{Host: "s3.cn-northwest-1.amazonaws.com.cn"}, + regionOverride: "", + expectedLocation: "cn-northwest-1", + }, + // No region provided, no standard region strings provided as well. - Test 6. { endpointURL: url.URL{Host: "s3.amazonaws.com"}, regionOverride: "", @@ -304,6 +312,7 @@ func TestIsStandardHeader(t *testing.T) { {"content-type", true}, {"cache-control", true}, {"content-disposition", true}, + {"content-language", true}, {"random-header", false}, } @@ -341,32 +350,6 @@ func TestIsSSEHeader(t *testing.T) { } } -// Tests if header is client encryption header -func TestIsCSEHeader(t *testing.T) { - testCases := []struct { - // Input. - header string - // Expected result. - expectedValue bool - }{ - {"x-amz-iv", true}, - {"x-amz-key", true}, - {"x-amz-matdesc", true}, - {"x-amz-meta-x-amz-iv", true}, - {"x-amz-meta-x-amz-key", true}, - {"x-amz-meta-x-amz-matdesc", true}, - {"random-header", false}, - } - - for i, testCase := range testCases { - actual := isCSEHeader(testCase.header) - if actual != testCase.expectedValue { - t.Errorf("Test %d: Expected to pass, but failed", i+1) - } - } - -} - // Tests if header is x-amz-meta or x-amz-acl func TestIsAmzHeader(t *testing.T) { testCases := []struct { -- cgit v1.2.3