diff options
| author | Félix Sipma <felix+debian@gueux.org> | 2019-11-23 20:49:18 +0100 |
|---|---|---|
| committer | Félix Sipma <felix+debian@gueux.org> | 2019-11-23 20:49:18 +0100 |
| commit | b75e2eba23b61af62951c5a5fc45f3eb4ffe68f3 (patch) | |
| tree | aa00eb410cd0f8f470823f6347ea3213283e93f0 | |
| parent | 70b5c53e64dae38106cf7e218cc99e6f493f12b9 (diff) | |
New upstream version 6.0.43
160 files changed, 5772 insertions, 1410 deletions
diff --git a/.travis.yml b/.travis.yml index 4ae1ead..e50cefe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,12 +6,9 @@ os: env: - ARCH=x86_64 -- ARCH=i686 go: -- 1.7.4 -- 1.8.x -- 1.9.x +- 1.13.x - tip matrix: @@ -19,12 +16,15 @@ matrix: allow_failures: - go: tip -addons: - apt: - packages: - - devscripts +before_install: + - sudo apt-get install devscripts + - mkdir /tmp/minio + - (cd /tmp/minio; GO111MODULE=on go get github.com/minio/minio) + - sudo cp testcerts/public.crt /usr/local/share/ca-certificates/ + - sudo update-ca-certificates + - MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 ${GOPATH}/bin/minio server --compat --quiet --certs-dir testcerts data 2>&1 > minio.log & script: -- diff -au <(gofmt -d .) <(printf "") -- diff -au <(licensecheck --check '.go$' --recursive --lines 0 * | grep -v -w 'Apache (v2.0)') <(printf "") -- make + - diff -au <(gofmt -d .) <(printf "") + - diff -au <(licensecheck --check '.go$' --recursive --lines 0 * | grep -v -w 'Apache (v2.0)') <(printf "") + - make diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 1797307..f640dfb 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -5,7 +5,7 @@ Please go through this link [Maintainer Responsibility](https://gist.github.com/abperiasamy/f4d9b31d3186bbd26522) ### Making new releases -Tag and sign your release commit, additionally this step requires you to have access to Minio's trusted private key. +Tag and sign your release commit, additionally this step requires you to have access to MinIO's trusted private key. ```sh $ export GNUPGHOME=/media/${USER}/minio/trusted $ git tag -s 4.0.0 @@ -23,11 +23,11 @@ $ grep libraryVersion api.go Commit your changes ``` -$ git commit -a -m "Update version for next release" --author "Minio Trusted <trusted@minio.io>" +$ git commit -a -m "Update version for next release" --author "MinIO Trusted <trusted@min.io>" ``` ### Announce -Announce new release by adding release notes at https://github.com/minio/minio-go/releases from `trusted@minio.io` account. Release notes requires two sections `highlights` and `changelog`. Highlights is a bulleted list of salient features in this release and Changelog contains list of all commits since the last release. +Announce new release by adding release notes at https://github.com/minio/minio-go/releases from `trusted@min.io` account. Release notes requires two sections `highlights` and `changelog`. Highlights is a bulleted list of salient features in this release and Changelog contains list of all commits since the last release. To generate `changelog` ```sh @@ -1,15 +1,22 @@ all: checks -checks: - @go get -t ./... - @go vet ./... - @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... - @go get github.com/dustin/go-humanize/... - @go get github.com/sirupsen/logrus/... - @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go +.PHONY: examples docs + +checks: vet test examples functional-test + +vet: + @GO111MODULE=on go vet ./... + +test: + @GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... + +examples: @mkdir -p /tmp/examples && for i in $(echo examples/s3/*); do go build -o /tmp/examples/$(basename ${i:0:-3}) ${i}; done - @go get -u github.com/a8m/mark/... - @go get -u github.com/minio/cli/... - @go get -u golang.org/x/tools/cmd/goimports - @go get -u github.com/gernest/wow/... - @go build docs/validator.go && ./validator -m docs/API.md -t docs/checker.go.tpl + +functional-test: + @GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go + +clean: + @echo "Cleaning up all the generated files" + @find . -name '*.test' | xargs rm -fv + @find . -name '*~' | xargs rm -fv @@ -1,2 +1,2 @@ minio-go -Copyright 2015-2017 Minio, Inc.
\ No newline at end of file +Copyright 2015-2017 MinIO, Inc.
\ No newline at end of file @@ -1,18 +1,18 @@ -# Minio Go Client SDK for Amazon S3 Compatible Cloud Storage [](https://slack.minio.io) [](https://sourcegraph.com/github.com/minio/minio-go?badge) [](https://github.com/minio/minio-go/blob/master/LICENSE) +# MinIO Go Client SDK for Amazon S3 Compatible Cloud Storage [](https://slack.min.io) [](https://sourcegraph.com/github.com/minio/minio-go?badge) [](https://github.com/minio/minio-go/blob/master/LICENSE) -The Minio Go Client SDK provides simple APIs to access any Amazon S3 compatible object storage. +The MinIO Go Client SDK provides simple APIs to access any Amazon S3 compatible object storage. -This quickstart guide will show you how to install the Minio client SDK, connect to Minio, and provide a walkthrough for a simple file uploader. For a complete list of APIs and examples, please take a look at the [Go Client API Reference](https://docs.minio.io/docs/golang-client-api-reference). +This quickstart guide will show you how to install the MinIO client SDK, connect to MinIO, and provide a walkthrough for a simple file uploader. For a complete list of APIs and examples, please take a look at the [Go Client API Reference](https://docs.min.io/docs/golang-client-api-reference). -This document assumes that you have a working [Go development environment](https://docs.minio.io/docs/how-to-install-golang). +This document assumes that you have a working [Go development environment](https://golang.org/doc/install). ## Download from Github ```sh -go get -u github.com/minio/minio-go +GO111MODULE=on go get github.com/minio/minio-go/v6 ``` -## Initialize Minio Client -Minio client requires the following four parameters specified to connect to an Amazon S3 compatible object storage. +## Initialize MinIO Client +MinIO client requires the following four parameters specified to connect to an Amazon S3 compatible object storage. | Parameter | Description| | :--- | :--- | @@ -26,12 +26,12 @@ Minio client requires the following four parameters specified to connect to an A package main import ( - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" "log" ) func main() { - endpoint := "play.minio.io:9000" + endpoint := "play.min.io" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true @@ -49,19 +49,19 @@ func main() { ## Quick Start Example - File Uploader This example program connects to an object storage server, creates a bucket and uploads a file to the bucket. -We will use the Minio server running at [https://play.minio.io:9000](https://play.minio.io:9000) in this example. Feel free to use this service for testing and development. Access credentials shown in this example are open to the public. +We will use the MinIO server running at [https://play.min.io](https://play.min.io) in this example. Feel free to use this service for testing and development. Access credentials shown in this example are open to the public. ### FileUploader.go ```go package main import ( - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" "log" ) func main() { - endpoint := "play.minio.io:9000" + endpoint := "play.min.io" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true @@ -79,8 +79,8 @@ func main() { err = minioClient.MakeBucket(bucketName, location) if err != nil { // Check to see if we already own this bucket (which happens if you run this twice) - exists, err := minioClient.BucketExists(bucketName) - if err == nil && exists { + exists, errBucketExists := minioClient.BucketExists(bucketName) + if errBucketExists == nil && exists { log.Printf("We already own %s\n", bucketName) } else { log.Fatalln(err) @@ -117,58 +117,58 @@ mc ls play/mymusic/ ## API Reference The full API Reference is available here. -* [Complete API Reference](https://docs.minio.io/docs/golang-client-api-reference) +* [Complete API Reference](https://docs.min.io/docs/golang-client-api-reference) ### API Reference : Bucket Operations -* [`MakeBucket`](https://docs.minio.io/docs/golang-client-api-reference#MakeBucket) -* [`ListBuckets`](https://docs.minio.io/docs/golang-client-api-reference#ListBuckets) -* [`BucketExists`](https://docs.minio.io/docs/golang-client-api-reference#BucketExists) -* [`RemoveBucket`](https://docs.minio.io/docs/golang-client-api-reference#RemoveBucket) -* [`ListObjects`](https://docs.minio.io/docs/golang-client-api-reference#ListObjects) -* [`ListObjectsV2`](https://docs.minio.io/docs/golang-client-api-reference#ListObjectsV2) -* [`ListIncompleteUploads`](https://docs.minio.io/docs/golang-client-api-reference#ListIncompleteUploads) +* [`MakeBucket`](https://docs.min.io/docs/golang-client-api-reference#MakeBucket) +* [`ListBuckets`](https://docs.min.io/docs/golang-client-api-reference#ListBuckets) +* [`BucketExists`](https://docs.min.io/docs/golang-client-api-reference#BucketExists) +* [`RemoveBucket`](https://docs.min.io/docs/golang-client-api-reference#RemoveBucket) +* [`ListObjects`](https://docs.min.io/docs/golang-client-api-reference#ListObjects) +* [`ListObjectsV2`](https://docs.min.io/docs/golang-client-api-reference#ListObjectsV2) +* [`ListIncompleteUploads`](https://docs.min.io/docs/golang-client-api-reference#ListIncompleteUploads) ### API Reference : Bucket policy Operations -* [`SetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketPolicy) -* [`GetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketPolicy) +* [`SetBucketPolicy`](https://docs.min.io/docs/golang-client-api-reference#SetBucketPolicy) +* [`GetBucketPolicy`](https://docs.min.io/docs/golang-client-api-reference#GetBucketPolicy) ### API Reference : Bucket notification Operations -* [`SetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketNotification) -* [`GetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketNotification) -* [`RemoveAllBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#RemoveAllBucketNotification) -* [`ListenBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#ListenBucketNotification) (Minio Extension) +* [`SetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#SetBucketNotification) +* [`GetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#GetBucketNotification) +* [`RemoveAllBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#RemoveAllBucketNotification) +* [`ListenBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenBucketNotification) (MinIO Extension) ### API Reference : File Object Operations -* [`FPutObject`](https://docs.minio.io/docs/golang-client-api-reference#FPutObject) -* [`FGetObject`](https://docs.minio.io/docs/golang-client-api-reference#FGetObject) -* [`FPutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FPutObjectWithContext) -* [`FGetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FGetObjectWithContext) +* [`FPutObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) +* [`FGetObject`](https://docs.min.io/docs/golang-client-api-reference#FGetObject) +* [`FPutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FPutObjectWithContext) +* [`FGetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FGetObjectWithContext) ### API Reference : Object Operations -* [`GetObject`](https://docs.minio.io/docs/golang-client-api-reference#GetObject) -* [`PutObject`](https://docs.minio.io/docs/golang-client-api-reference#PutObject) -* [`GetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#GetObjectWithContext) -* [`PutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#PutObjectWithContext) -* [`PutObjectStreaming`](https://docs.minio.io/docs/golang-client-api-reference#PutObjectStreaming) -* [`StatObject`](https://docs.minio.io/docs/golang-client-api-reference#StatObject) -* [`CopyObject`](https://docs.minio.io/docs/golang-client-api-reference#CopyObject) -* [`RemoveObject`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObject) -* [`RemoveObjects`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObjects) -* [`RemoveIncompleteUpload`](https://docs.minio.io/docs/golang-client-api-reference#RemoveIncompleteUpload) -* [`SelectObjectContent`](https://docs.minio.io/docs/golang-client-api-reference#SelectObjectContent) +* [`GetObject`](https://docs.min.io/docs/golang-client-api-reference#GetObject) +* [`PutObject`](https://docs.min.io/docs/golang-client-api-reference#PutObject) +* [`GetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#GetObjectWithContext) +* [`PutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#PutObjectWithContext) +* [`PutObjectStreaming`](https://docs.min.io/docs/golang-client-api-reference#PutObjectStreaming) +* [`StatObject`](https://docs.min.io/docs/golang-client-api-reference#StatObject) +* [`CopyObject`](https://docs.min.io/docs/golang-client-api-reference#CopyObject) +* [`RemoveObject`](https://docs.min.io/docs/golang-client-api-reference#RemoveObject) +* [`RemoveObjects`](https://docs.min.io/docs/golang-client-api-reference#RemoveObjects) +* [`RemoveIncompleteUpload`](https://docs.min.io/docs/golang-client-api-reference#RemoveIncompleteUpload) +* [`SelectObjectContent`](https://docs.min.io/docs/golang-client-api-reference#SelectObjectContent) ### API Reference : Presigned Operations -* [`PresignedGetObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedGetObject) -* [`PresignedPutObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedPutObject) -* [`PresignedHeadObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedHeadObject) -* [`PresignedPostPolicy`](https://docs.minio.io/docs/golang-client-api-reference#PresignedPostPolicy) +* [`PresignedGetObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedGetObject) +* [`PresignedPutObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedPutObject) +* [`PresignedHeadObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedHeadObject) +* [`PresignedPostPolicy`](https://docs.min.io/docs/golang-client-api-reference#PresignedPostPolicy) ### API Reference : Client custom settings -* [`SetAppInfo`](http://docs.minio.io/docs/golang-client-api-reference#SetAppInfo) -* [`SetCustomTransport`](http://docs.minio.io/docs/golang-client-api-reference#SetCustomTransport) -* [`TraceOn`](http://docs.minio.io/docs/golang-client-api-reference#TraceOn) -* [`TraceOff`](http://docs.minio.io/docs/golang-client-api-reference#TraceOff) +* [`SetAppInfo`](http://docs.min.io/docs/golang-client-api-reference#SetAppInfo) +* [`SetCustomTransport`](http://docs.min.io/docs/golang-client-api-reference#SetCustomTransport) +* [`TraceOn`](http://docs.min.io/docs/golang-client-api-reference#TraceOn) +* [`TraceOff`](http://docs.min.io/docs/golang-client-api-reference#TraceOff) ## Full Examples @@ -194,7 +194,7 @@ The full API Reference is available here. * [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) * [removeallbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeallbucketnotification.go) -* [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (Minio Extension) +* [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (MinIO Extension) ### Full Examples : File Object Operations * [fputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject.go) @@ -225,9 +225,8 @@ The full API Reference is available here. * [presignedpostpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedpostpolicy.go) ## Explore Further -* [Complete Documentation](https://docs.minio.io) -* [Minio Go Client SDK API Reference](https://docs.minio.io/docs/golang-client-api-reference) -* [Go Music Player App Full Application Example](https://docs.minio.io/docs/go-music-player-app) +* [Complete Documentation](https://docs.min.io) +* [MinIO Go Client SDK API Reference](https://docs.min.io/docs/golang-client-api-reference) ## Contribute [Contributors Guide](https://github.com/minio/minio-go/blob/master/CONTRIBUTING.md) diff --git a/README_zh_CN.md b/README_zh_CN.md index a5acf19..b5ba0c1 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -1,12 +1,12 @@ -# 适用于与Amazon S3兼容云存储的Minio Go SDK [](https://slack.minio.io) [](https://sourcegraph.com/github.com/minio/minio-go?badge) +# 适用于与Amazon S3兼容云存储的MinIO Go SDK [](https://slack.min.io) [](https://sourcegraph.com/github.com/minio/minio-go?badge) -Minio Go Client SDK提供了简单的API来访问任何与Amazon S3兼容的对象存储服务。 +MinIO Go Client SDK提供了简单的API来访问任何与Amazon S3兼容的对象存储服务。 **支持的云存储:** - AWS Signature Version 4 - Amazon S3 - - Minio + - MinIO - AWS Signature Version 2 - Google Cloud Storage (兼容模式) @@ -14,17 +14,17 @@ Minio Go Client SDK提供了简单的API来访问任何与Amazon S3兼容的对 - Ceph Object Gateway - Riak CS -本文我们将学习如何安装Minio client SDK，连接到Minio，并提供一下文件上传的示例。对于完整的API以及示例，请参考[Go Client API Reference](https://docs.minio.io/docs/golang-client-api-reference)。 +本文我们将学习如何安装MinIO client SDK，连接到MinIO，并提供一下文件上传的示例。对于完整的API以及示例，请参考[Go Client API Reference](https://docs.min.io/docs/golang-client-api-reference)。 -本文假设你已经有 [Go开发环境](https://docs.minio.io/docs/how-to-install-golang)。 +本文假设你已经有 [Go开发环境](https://golang.org/doc/install)。 ## 从Github下载 ```sh go get -u github.com/minio/minio-go ``` -## 初始化Minio Client -Minio client需要以下4个参数来连接与Amazon S3兼容的对象存储。 +## 初始化MinIO Client +MinIO client需要以下4个参数来连接与Amazon S3兼容的对象存储。 | 参数 | 描述| | :--- | :--- | @@ -38,12 +38,12 @@ Minio client需要以下4个参数来连接与Amazon S3兼容的对象存储。 package main import ( - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" "log" ) func main() { - endpoint := "play.minio.io:9000" + endpoint := "play.min.io" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true @@ -61,19 +61,19 @@ func main() { ## 示例-文件上传 本示例连接到一个对象存储服务，创建一个存储桶并上传一个文件到存储桶中。 -我们在本示例中使用运行在 [https://play.minio.io:9000](https://play.minio.io:9000) 上的Minio服务，你可以用这个服务来开发和测试。示例中的访问凭据是公开的。 +我们在本示例中使用运行在 [https://play.min.io](https://play.min.io) 上的MinIO服务，你可以用这个服务来开发和测试。示例中的访问凭据是公开的。 ### FileUploader.go ```go package main import ( - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" "log" ) func main() { - endpoint := "play.minio.io:9000" + endpoint := "play.min.io" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true @@ -127,60 +127,60 @@ mc ls play/mymusic/ ## API文档 完整的API文档在这里。 -* [完整API文档](https://docs.minio.io/docs/golang-client-api-reference) +* [完整API文档](https://docs.min.io/docs/golang-client-api-reference) ### API文档 : 操作存储桶 -* [`MakeBucket`](https://docs.minio.io/docs/golang-client-api-reference#MakeBucket) -* [`ListBuckets`](https://docs.minio.io/docs/golang-client-api-reference#ListBuckets) -* [`BucketExists`](https://docs.minio.io/docs/golang-client-api-reference#BucketExists) -* [`RemoveBucket`](https://docs.minio.io/docs/golang-client-api-reference#RemoveBucket) -* [`ListObjects`](https://docs.minio.io/docs/golang-client-api-reference#ListObjects) -* [`ListObjectsV2`](https://docs.minio.io/docs/golang-client-api-reference#ListObjectsV2) -* [`ListIncompleteUploads`](https://docs.minio.io/docs/golang-client-api-reference#ListIncompleteUploads) +* [`MakeBucket`](https://docs.min.io/docs/golang-client-api-reference#MakeBucket) +* [`ListBuckets`](https://docs.min.io/docs/golang-client-api-reference#ListBuckets) +* [`BucketExists`](https://docs.min.io/docs/golang-client-api-reference#BucketExists) +* [`RemoveBucket`](https://docs.min.io/docs/golang-client-api-reference#RemoveBucket) +* [`ListObjects`](https://docs.min.io/docs/golang-client-api-reference#ListObjects) +* [`ListObjectsV2`](https://docs.min.io/docs/golang-client-api-reference#ListObjectsV2) +* [`ListIncompleteUploads`](https://docs.min.io/docs/golang-client-api-reference#ListIncompleteUploads) ### API文档 : 存储桶策略 -* [`SetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketPolicy) -* [`GetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketPolicy) +* [`SetBucketPolicy`](https://docs.min.io/docs/golang-client-api-reference#SetBucketPolicy) +* [`GetBucketPolicy`](https://docs.min.io/docs/golang-client-api-reference#GetBucketPolicy) ### API文档 : 存储桶通知 -* [`SetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketNotification) -* [`GetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketNotification) -* [`RemoveAllBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#RemoveAllBucketNotification) -* [`ListenBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#ListenBucketNotification) (Minio Extension) +* [`SetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#SetBucketNotification) +* [`GetBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#GetBucketNotification) +* [`RemoveAllBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#RemoveAllBucketNotification) +* [`ListenBucketNotification`](https://docs.min.io/docs/golang-client-api-reference#ListenBucketNotification) (MinIO Extension) ### API文档 : 操作文件对象 -* [`FPutObject`](https://docs.minio.io/docs/golang-client-api-reference#FPutObject) -* [`FGetObject`](https://docs.minio.io/docs/golang-client-api-reference#FPutObject) -* [`FPutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FPutObjectWithContext) -* [`FGetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FGetObjectWithContext) +* [`FPutObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) +* [`FGetObject`](https://docs.min.io/docs/golang-client-api-reference#FPutObject) +* [`FPutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FPutObjectWithContext) +* [`FGetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#FGetObjectWithContext) ### API文档 : 操作对象 -* [`GetObject`](https://docs.minio.io/docs/golang-client-api-reference#GetObject) -* [`PutObject`](https://docs.minio.io/docs/golang-client-api-reference#PutObject) -* [`GetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#GetObjectWithContext) -* [`PutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#PutObjectWithContext) -* [`PutObjectStreaming`](https://docs.minio.io/docs/golang-client-api-reference#PutObjectStreaming) -* [`StatObject`](https://docs.minio.io/docs/golang-client-api-reference#StatObject) -* [`CopyObject`](https://docs.minio.io/docs/golang-client-api-reference#CopyObject) -* [`RemoveObject`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObject) -* [`RemoveObjects`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObjects) -* [`RemoveIncompleteUpload`](https://docs.minio.io/docs/golang-client-api-reference#RemoveIncompleteUpload) +* [`GetObject`](https://docs.min.io/docs/golang-client-api-reference#GetObject) +* [`PutObject`](https://docs.min.io/docs/golang-client-api-reference#PutObject) +* [`GetObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#GetObjectWithContext) +* [`PutObjectWithContext`](https://docs.min.io/docs/golang-client-api-reference#PutObjectWithContext) +* [`PutObjectStreaming`](https://docs.min.io/docs/golang-client-api-reference#PutObjectStreaming) +* [`StatObject`](https://docs.min.io/docs/golang-client-api-reference#StatObject) +* [`CopyObject`](https://docs.min.io/docs/golang-client-api-reference#CopyObject) +* [`RemoveObject`](https://docs.min.io/docs/golang-client-api-reference#RemoveObject) +* [`RemoveObjects`](https://docs.min.io/docs/golang-client-api-reference#RemoveObjects) +* [`RemoveIncompleteUpload`](https://docs.min.io/docs/golang-client-api-reference#RemoveIncompleteUpload) ### API文档: 操作加密对象 -* [`GetEncryptedObject`](https://docs.minio.io/docs/golang-client-api-reference#GetEncryptedObject) -* [`PutEncryptedObject`](https://docs.minio.io/docs/golang-client-api-reference#PutEncryptedObject) +* [`GetEncryptedObject`](https://docs.min.io/docs/golang-client-api-reference#GetEncryptedObject) +* [`PutEncryptedObject`](https://docs.min.io/docs/golang-client-api-reference#PutEncryptedObject) ### API文档 : Presigned操作 -* [`PresignedGetObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedGetObject) -* [`PresignedPutObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedPutObject) -* [`PresignedHeadObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedHeadObject) -* [`PresignedPostPolicy`](https://docs.minio.io/docs/golang-client-api-reference#PresignedPostPolicy) +* [`PresignedGetObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedGetObject) +* [`PresignedPutObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedPutObject) +* [`PresignedHeadObject`](https://docs.min.io/docs/golang-client-api-reference#PresignedHeadObject) +* [`PresignedPostPolicy`](https://docs.min.io/docs/golang-client-api-reference#PresignedPostPolicy) ### API文档 : 客户端自定义设置 -* [`SetAppInfo`](http://docs.minio.io/docs/golang-client-api-reference#SetAppInfo) -* [`SetCustomTransport`](http://docs.minio.io/docs/golang-client-api-reference#SetCustomTransport) -* [`TraceOn`](http://docs.minio.io/docs/golang-client-api-reference#TraceOn) -* [`TraceOff`](http://docs.minio.io/docs/golang-client-api-reference#TraceOff) +* [`SetAppInfo`](http://docs.min.io/docs/golang-client-api-reference#SetAppInfo) +* [`SetCustomTransport`](http://docs.min.io/docs/golang-client-api-reference#SetCustomTransport) +* [`TraceOn`](http://docs.min.io/docs/golang-client-api-reference#TraceOn) +* [`TraceOff`](http://docs.min.io/docs/golang-client-api-reference#TraceOff) ## 完整示例 @@ -202,7 +202,7 @@ mc ls play/mymusic/ * [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) * [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) * [removeallbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeallbucketnotification.go) -* [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (Minio扩展) +* [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (MinIO扩展) ### 完整示例 : 操作文件对象 * [fputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject.go) @@ -233,9 +233,8 @@ mc ls play/mymusic/ * [presignedpostpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedpostpolicy.go) ## 了解更多 -* [完整文档](https://docs.minio.io) -* [Minio Go Client SDK API文档](https://docs.minio.io/docs/golang-client-api-reference) -* [Go 音乐播放器完整示例](https://docs.minio.io/docs/go-music-player-app) +* [完整文档](https://docs.min.io) +* [MinIO Go Client SDK API文档](https://docs.min.io/docs/golang-client-api-reference) ## 贡献 [贡献指南](https://github.com/minio/minio-go/blob/master/docs/zh_CN/CONTRIBUTING.md) diff --git a/api-compose-object.go b/api-compose-object.go index 3ac36c5..748b558 100644 --- a/api-compose-object.go +++ b/api-compose-object.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017, 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017, 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,8 +28,8 @@ import ( "strings" "time" - "github.com/minio/minio-go/pkg/encrypt" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // DestinationInfo - type with information about the object to be diff --git a/api-compose-object_test.go b/api-compose-object_test.go index 295bbc2..e9f04d3 100644 --- a/api-compose-object_test.go +++ b/api-compose-object_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-datatypes.go b/api-datatypes.go index 63fc089..4e54f34 100644 --- a/api-datatypes.go +++ b/api-datatypes.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ package minio import ( + "encoding/xml" + "io" "net/http" "time" ) @@ -30,6 +32,36 @@ type BucketInfo struct { CreationDate time.Time `json:"creationDate"` } +// StringMap represents map with custom UnmarshalXML +type StringMap map[string]string + +// UnmarshalXML unmarshals the XML into a map of string to strings, +// creating a key in the map for each tag and setting it's value to the +// tags contents. +// +// The fact this function is on the pointer of Map is important, so that +// if m is nil it can be initialized, which is often the case if m is +// nested in another xml structurel. This is also why the first thing done +// on the first line is initialize it. +func (m *StringMap) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + *m = StringMap{} + type xmlMapEntry struct { + XMLName xml.Name + Value string `xml:",chardata"` + } + for { + var e xmlMapEntry + err := d.Decode(&e) + if err == io.EOF { + break + } else if err != nil { + return err + } + (*m)[e.XMLName.Local] = e.Value + } + return nil +} + // ObjectInfo container for object metadata. type ObjectInfo struct { // An ETag is optionally set to md5sum of an object. In case of multipart objects, @@ -41,11 +73,15 @@ type ObjectInfo struct { LastModified time.Time `json:"lastModified"` // Date and time the object was last modified. Size int64 `json:"size"` // Size in bytes of the object. ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data. + Expires time.Time `json:"expires"` // The date and time at which the object is no longer able to be cached. // Collection of additional metadata on the object. // eg: x-amz-meta-*, content-encoding etc. Metadata http.Header `json:"metadata" xml:"-"` + // x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value. + UserMetadata StringMap `json:"userMetadata"` + // Owner name. Owner struct { DisplayName string `json:"name"` diff --git a/api-error-response.go b/api-error-response.go index 655991c..726fdd2 100644 --- a/api-error-response.go +++ b/api-error-response.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,6 +36,8 @@ import ( */ // ErrorResponse - Is the typed error returned by all API operations. +// ErrorResponse struct should be comparable since it is compared inside +// golang http API (https://github.com/golang/go/issues/29768) type ErrorResponse struct { XMLName xml.Name `xml:"Error" json:"-"` Code string @@ -51,9 +53,6 @@ type ErrorResponse struct { // Underlying HTTP status code for the returned error StatusCode int `xml:"-" json:"-"` - - // Headers of the returned S3 XML error - Headers http.Header `xml:"-" json:"-"` } // ToErrorResponse - Returns parsed ErrorResponse struct from body and @@ -61,7 +60,7 @@ type ErrorResponse struct { // // For example: // -// import s3 "github.com/minio/minio-go" +// import s3 "github.com/minio/minio-go/v6" // ... // ... // reader, stat, err := s3.GetObject(...) @@ -177,9 +176,6 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) errResp.Message = fmt.Sprintf("Region does not match, expecting region ‘%s’.", errResp.Region) } - // Save headers returned in the API XML error - errResp.Headers = resp.Header - return errResp } diff --git a/api-error-response_test.go b/api-error-response_test.go index bf10941..0111bd9 100644 --- a/api-error-response_test.go +++ b/api-error-response_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -77,7 +77,6 @@ func TestHttpRespToErrorResponse(t *testing.T) { RequestID: resp.Header.Get("x-amz-request-id"), HostID: resp.Header.Get("x-amz-id-2"), Region: resp.Header.Get("x-amz-bucket-region"), - Headers: resp.Header, } return errResp } @@ -292,3 +291,13 @@ func TestErrWithoutMessage(t *testing.T) { t.Errorf("Expected \"Error response code InvalidArgument.\", got %s", errResp) } } + +// Tests if ErrorResponse is comparable since it is compared +// inside golang http code (https://github.com/golang/go/issues/29768) +func TestErrorResponseComparable(t *testing.T) { + var e1 interface{} = ErrorResponse{} + var e2 interface{} = ErrorResponse{} + if e1 != e2 { + t.Fatalf("ErrorResponse should be comparable") + } +} diff --git a/api-get-lifecycle.go b/api-get-lifecycle.go index 8097bfc..a24d03e 100644 --- a/api-get-lifecycle.go +++ b/api-get-lifecycle.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // GetBucketLifecycle - get bucket lifecycle. diff --git a/api-get-object-acl.go b/api-get-object-acl.go index af5544d..ea80949 100644 --- a/api-get-object-acl.go +++ b/api-get-object-acl.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-get-object-context.go b/api-get-object-context.go index f8dfac7..3c3afd0 100644 --- a/api-get-object-context.go +++ b/api-get-object-context.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-get-object-file.go b/api-get-object-file.go index a852220..0684b2b 100644 --- a/api-get-object-file.go +++ b/api-get-object-file.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "os" "path/filepath" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // FGetObjectWithContext - download contents of an object to a local file. @@ -100,7 +100,7 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam } // Seek to current position for incoming reader. - objectReader, objectStat, err := c.getObject(ctx, bucketName, objectName, opts) + objectReader, objectStat, _, err := c.getObject(ctx, bucketName, objectName, opts) if err != nil { return err } diff --git a/api-get-object.go b/api-get-object.go index 0bf556e..1367828 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ import ( "sync" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // GetObject - returns an seekable, readable object. @@ -92,7 +92,7 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName } else if req.Offset > 0 { opts.SetRange(req.Offset, 0) } - httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, opts) + httpReader, objectInfo, _, err = c.getObject(ctx, bucketName, objectName, opts) if err != nil { resCh <- getResponse{Error: err} return @@ -173,7 +173,7 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName } else if req.Offset > 0 { // Range is set with respect to the offset. opts.SetRange(req.Offset, 0) } - httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, opts) + httpReader, objectInfo, _, err = c.getObject(ctx, bucketName, objectName, opts) if err != nil { resCh <- getResponse{ Error: err, @@ -321,6 +321,7 @@ func (o *Object) Read(b []byte) (n int, err error) { if o.prevErr != nil || o.isClosed { return 0, o.prevErr } + // Create a new request. readReq := getRequest{ isReadOp: true, @@ -403,10 +404,13 @@ func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { defer o.mutex.Unlock() // prevErr is error which was saved in previous operation. - if o.prevErr != nil || o.isClosed { + if o.prevErr != nil && o.prevErr != io.EOF || o.isClosed { return 0, o.prevErr } + // Set the current offset to ReadAt offset, because the current offset will be shifted at the end of this method. + o.currOffset = offset + // Can only compare offsets to size when size has been set. if o.objectInfoSet { // If offset is negative than we return io.EOF. @@ -476,11 +480,9 @@ func (o *Object) Seek(offset int64, whence int) (n int64, err error) { o.mutex.Lock() defer o.mutex.Unlock() - if o.prevErr != nil { - // At EOF seeking is legal allow only io.EOF, for any other errors we return. - if o.prevErr != io.EOF { - return 0, o.prevErr - } + // At EOF seeking is legal allow only io.EOF, for any other errors we return. + if o.prevErr != nil && o.prevErr != io.EOF { + return 0, o.prevErr } // Negative offset is valid for whence of '2'. @@ -594,13 +596,13 @@ func newObject(reqCh chan<- getRequest, resCh <-chan getResponse, doneCh chan<- // // For more information about the HTTP Range header. // go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. -func (c Client) getObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, error) { +func (c Client) getObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { // Validate input arguments. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return nil, ObjectInfo{}, err + return nil, ObjectInfo{}, nil, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { - return nil, ObjectInfo{}, err + return nil, ObjectInfo{}, nil, err } // Execute GET on objectName. @@ -611,11 +613,11 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op contentSHA256Hex: emptySHA256Hex, }) if err != nil { - return nil, ObjectInfo{}, err + return nil, ObjectInfo{}, nil, err } if resp != nil { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - return nil, ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) + return nil, ObjectInfo{}, nil, httpRespToErrorResponse(resp, bucketName, objectName) } } @@ -627,7 +629,7 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) if err != nil { msg := "Last-Modified time format not recognized. " + reportIssue - return nil, ObjectInfo{}, ErrorResponse{ + return nil, ObjectInfo{}, nil, ErrorResponse{ Code: "InternalError", Message: msg, RequestID: resp.Header.Get("x-amz-request-id"), @@ -655,5 +657,5 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op } // do not close body here, caller will close - return resp.Body, objectStat, nil + return resp.Body, objectStat, resp.Header, nil } diff --git a/api-get-options.go b/api-get-options.go index dbf062d..538fd1a 100644 --- a/api-get-options.go +++ b/api-get-options.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ import ( "net/http" "time" - "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/encrypt" ) // GetObjectOptions are used to specify additional headers or options diff --git a/api-get-policy.go b/api-get-policy.go index 12d4c59..bc1d105 100644 --- a/api-get-policy.go +++ b/api-get-policy.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // GetBucketPolicy - get bucket policy at a given path. diff --git a/api-list.go b/api-list.go index 04f7573..09c1565 100644 --- a/api-list.go +++ b/api-list.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import ( "net/url" "strings" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // ListBuckets list all buckets owned by this authenticated user. @@ -60,9 +60,13 @@ func (c Client) ListBuckets() ([]BucketInfo, error) { /// Bucket Read Operations. -// ListObjectsV2 lists all objects matching the objectPrefix from -// the specified bucket. If recursion is enabled it would list -// all subdirectories and all its contents. +// ListObjectsV2WithMetadata lists all objects matching the objectPrefix +// from the specified bucket. If recursion is enabled it would list +// all subdirectories and all its contents. This call adds +// UserMetadata information as well for each object. +// +// This is a MinIO extension, this will not work against other S3 +// compatible object storage vendors. // // Your input parameters are just bucketName, objectPrefix, recursive // and a done channel for pro-actively closing the internal go @@ -76,11 +80,18 @@ func (c Client) ListBuckets() ([]BucketInfo, error) { // defer close(doneCh) // // Recursively list all objects in 'mytestbucket' // recursive := true -// for message := range api.ListObjectsV2("mytestbucket", "starthere", recursive, doneCh) { +// // Add metadata +// metadata := true +// for message := range api.ListObjectsV2WithMetadata("mytestbucket", "starthere", recursive, doneCh) { // fmt.Println(message) // } // -func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { +func (c Client) ListObjectsV2WithMetadata(bucketName, objectPrefix string, recursive bool, + doneCh <-chan struct{}) <-chan ObjectInfo { + return c.listObjectsV2(bucketName, objectPrefix, recursive, true, doneCh) +} + +func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metadata bool, doneCh <-chan struct{}) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" @@ -118,7 +129,8 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d var continuationToken string for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, 1000, "") + result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, + fetchOwner, metadata, delimiter, 0, "") if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -142,10 +154,7 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d for _, obj := range result.CommonPrefixes { select { // Send object prefixes. - case objectStatCh <- ObjectInfo{ - Key: obj.Prefix, - Size: 0, - }: + case objectStatCh <- ObjectInfo{Key: obj.Prefix}: // If receives done from the caller, return here. case <-doneCh: return @@ -166,6 +175,30 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d return objectStatCh } +// ListObjectsV2 lists all objects matching the objectPrefix from +// the specified bucket. If recursion is enabled it would list +// all subdirectories and all its contents. +// +// Your input parameters are just bucketName, objectPrefix, recursive +// and a done channel for pro-actively closing the internal go +// routine. If you enable recursive as 'true' this function will +// return back all the objects in a given bucket name and object +// prefix. +// +// api := client.New(....) +// // Create a done channel. +// doneCh := make(chan struct{}) +// defer close(doneCh) +// // Recursively list all objects in 'mytestbucket' +// recursive := true +// for message := range api.ListObjectsV2("mytestbucket", "starthere", recursive, doneCh) { +// fmt.Println(message) +// } +// +func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { + return c.listObjectsV2(bucketName, objectPrefix, recursive, false, doneCh) +} + // listObjectsV2Query - (List Objects V2) - List some or all (up to 1000) of the objects in a bucket. // // You can use the request parameters as selection criteria to return a subset of the objects in a bucket. @@ -176,7 +209,8 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. // ?start-after - Specifies the key to start after when listing objects in a bucket. -func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { +// ?metadata - Specifies if we want metadata for the objects as part of list operation. +func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketV2Result{}, err @@ -192,30 +226,33 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s // Always set list-type in ListObjects V2 urlValues.Set("list-type", "2") - // Set object prefix. - if objectPrefix != "" { - urlValues.Set("prefix", objectPrefix) + if metadata { + urlValues.Set("metadata", "true") } + + // Always set encoding-type in ListObjects V2 + urlValues.Set("encoding-type", "url") + + // Set object prefix, prefix value to be set to empty is okay. + urlValues.Set("prefix", objectPrefix) + + // Set delimiter, delimiter value to be set to empty is okay. + urlValues.Set("delimiter", delimiter) + // Set continuation token if continuationToken != "" { urlValues.Set("continuation-token", continuationToken) } - // Set delimiter. - if delimiter != "" { - urlValues.Set("delimiter", delimiter) - } // Fetch owner when listing if fetchOwner { urlValues.Set("fetch-owner", "true") } - // maxkeys should default to 1000 or less. - if maxkeys == 0 || maxkeys > 1000 { - maxkeys = 1000 - } // Set max keys. - urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + if maxkeys > 0 { + urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + } // Set start-after if startAfter != "" { @@ -250,6 +287,20 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s return listBucketResult, errors.New("Truncated response should have continuation token set") } + for i, obj := range listBucketResult.Contents { + listBucketResult.Contents[i].Key, err = url.QueryUnescape(obj.Key) + if err != nil { + return listBucketResult, err + } + } + + for i, obj := range listBucketResult.CommonPrefixes { + listBucketResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) + if err != nil { + return listBucketResult, err + } + } + // Success. return listBucketResult, nil } @@ -309,7 +360,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don var marker string for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 1000) + result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 0) if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -333,12 +384,9 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { - object := ObjectInfo{} - object.Key = obj.Prefix - object.Size = 0 select { // Send object prefixes. - case objectStatCh <- object: + case objectStatCh <- ObjectInfo{Key: obj.Prefix}: // If receives done from the caller, return here. case <-doneCh: return @@ -380,25 +428,25 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) - // Set object prefix. - if objectPrefix != "" { - urlValues.Set("prefix", objectPrefix) - } + + // Set object prefix, prefix value to be set to empty is okay. + urlValues.Set("prefix", objectPrefix) + + // Set delimiter, delimiter value to be set to empty is okay. + urlValues.Set("delimiter", delimiter) + // Set object marker. if objectMarker != "" { urlValues.Set("marker", objectMarker) } - // Set delimiter. - if delimiter != "" { - urlValues.Set("delimiter", delimiter) - } - // maxkeys should default to 1000 or less. - if maxkeys == 0 || maxkeys > 1000 { - maxkeys = 1000 - } // Set max keys. - urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + if maxkeys > 0 { + urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + } + + // Always set encoding-type + urlValues.Set("encoding-type", "url") // Execute GET on bucket to list objects. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ @@ -421,6 +469,28 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit if err != nil { return listBucketResult, err } + + for i, obj := range listBucketResult.Contents { + listBucketResult.Contents[i].Key, err = url.QueryUnescape(obj.Key) + if err != nil { + return listBucketResult, err + } + } + + for i, obj := range listBucketResult.CommonPrefixes { + listBucketResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) + if err != nil { + return listBucketResult, err + } + } + + if listBucketResult.NextMarker != "" { + listBucketResult.NextMarker, err = url.QueryUnescape(listBucketResult.NextMarker) + if err != nil { + return listBucketResult, err + } + } + return listBucketResult, nil } @@ -484,16 +554,16 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive var uploadIDMarker string for { // list all multipart uploads. - result, err := c.listMultipartUploadsQuery(bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 1000) + result, err := c.listMultipartUploadsQuery(bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 0) if err != nil { objectMultipartStatCh <- ObjectMultipartInfo{ Err: err, } return } - // Save objectMarker and uploadIDMarker for next request. objectMarker = result.NextKeyMarker uploadIDMarker = result.NextUploadIDMarker + // Send all multipart uploads. for _, obj := range result.Uploads { // Calculate total size of the uploaded parts if 'aggregateSize' is enabled. @@ -518,12 +588,9 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { - object := ObjectMultipartInfo{} - object.Key = obj.Prefix - object.Size = 0 select { // Send delimited prefixes here. - case objectMultipartStatCh <- object: + case objectMultipartStatCh <- ObjectMultipartInfo{Key: obj.Prefix, Size: 0}: // If done channel return here. case <-doneCh: return @@ -563,21 +630,21 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, if uploadIDMarker != "" { urlValues.Set("upload-id-marker", uploadIDMarker) } - // Set prefix marker. - if prefix != "" { - urlValues.Set("prefix", prefix) - } - // Set delimiter. - if delimiter != "" { - urlValues.Set("delimiter", delimiter) - } + + // Set object prefix, prefix value to be set to empty is okay. + urlValues.Set("prefix", prefix) + + // Set delimiter, delimiter value to be set to empty is okay. + urlValues.Set("delimiter", delimiter) + + // Always set encoding-type + urlValues.Set("encoding-type", "url") // maxUploads should be 1000 or less. - if maxUploads == 0 || maxUploads > 1000 { - maxUploads = 1000 + if maxUploads > 0 { + // Set max-uploads. + urlValues.Set("max-uploads", fmt.Sprintf("%d", maxUploads)) } - // Set max-uploads. - urlValues.Set("max-uploads", fmt.Sprintf("%d", maxUploads)) // Execute GET on bucketName to list multipart uploads. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ @@ -600,6 +667,31 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, if err != nil { return listMultipartUploadsResult, err } + + listMultipartUploadsResult.NextKeyMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextKeyMarker) + if err != nil { + return listMultipartUploadsResult, err + } + + listMultipartUploadsResult.NextUploadIDMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextUploadIDMarker) + if err != nil { + return listMultipartUploadsResult, err + } + + for i, obj := range listMultipartUploadsResult.Uploads { + listMultipartUploadsResult.Uploads[i].Key, err = url.QueryUnescape(obj.Key) + if err != nil { + return listMultipartUploadsResult, err + } + } + + for i, obj := range listMultipartUploadsResult.CommonPrefixes { + listMultipartUploadsResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) + if err != nil { + return listMultipartUploadsResult, err + } + } + return listMultipartUploadsResult, nil } @@ -688,11 +780,10 @@ func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, pa urlValues.Set("uploadId", uploadID) // maxParts should be 1000 or less. - if maxParts == 0 || maxParts > 1000 { - maxParts = 1000 + if maxParts > 0 { + // Set max parts. + urlValues.Set("max-parts", fmt.Sprintf("%d", maxParts)) } - // Set max parts. - urlValues.Set("max-parts", fmt.Sprintf("%d", maxParts)) // Execute GET on objectName to get list of parts. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ diff --git a/api-notification.go b/api-notification.go index 1c01e36..0480c21 100644 --- a/api-notification.go +++ b/api-notification.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,12 +21,11 @@ import ( "bufio" "context" "encoding/json" - "io" "net/http" "net/url" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // GetBucketNotification - get bucket notification at a given path. @@ -164,13 +163,14 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Indicate to our routine to exit cleanly upon return. defer close(retryDoneCh) + // Prepare urlValues to pass into the request on every loop + urlValues := make(url.Values) + urlValues.Set("prefix", prefix) + urlValues.Set("suffix", suffix) + urlValues["events"] = events + // Wait on the jitter retry loop. for range c.newRetryTimerContinous(time.Second, time.Second*30, MaxJitter, retryDoneCh) { - urlValues := make(url.Values) - urlValues.Set("prefix", prefix) - urlValues.Set("suffix", suffix) - urlValues["events"] = events - // Execute GET on bucket to list objects. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ bucketName: bucketName, @@ -196,30 +196,33 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Initialize a new bufio scanner, to read line by line. bio := bufio.NewScanner(resp.Body) - // Close the response body. - defer resp.Body.Close() - // Unmarshal each line, returns marshalled values. for bio.Scan() { var notificationInfo NotificationInfo if err = json.Unmarshal(bio.Bytes(), ¬ificationInfo); err != nil { + // Unexpected error during json unmarshal, send + // the error to caller for actionable as needed. + notificationInfoCh <- NotificationInfo{ + Err: err, + } + closeResponse(resp) continue } // Send notificationInfo select { case notificationInfoCh <- notificationInfo: case <-doneCh: + closeResponse(resp) return } } - // Look for any underlying errors. if err = bio.Err(); err != nil { - // For an unexpected connection drop from server, we close the body - // and re-connect. - if err == io.ErrUnexpectedEOF { - resp.Body.Close() + notificationInfoCh <- NotificationInfo{ + Err: err, } } + // Close current connection before looping further. + closeResponse(resp) } }(notificationInfoCh) diff --git a/api-object-lock.go b/api-object-lock.go new file mode 100644 index 0000000..c30ab32 --- /dev/null +++ b/api-object-lock.go @@ -0,0 +1,232 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/v6/pkg/s3utils" +) + +// RetentionMode - object retention mode. +type RetentionMode string + +const ( + // Governance - goverance mode. + Governance RetentionMode = "GOVERNANCE" + + // Compliance - compliance mode. + Compliance RetentionMode = "COMPLIANCE" +) + +func (r RetentionMode) String() string { + return string(r) +} + +// IsValid - check whether this retention mode is valid or not. +func (r RetentionMode) IsValid() bool { + return r == Governance || r == Compliance +} + +// ValidityUnit - retention validity unit. +type ValidityUnit string + +const ( + // Days - denotes no. of days. + Days ValidityUnit = "DAYS" + + // Years - denotes no. of years. + Years ValidityUnit = "YEARS" +) + +func (unit ValidityUnit) String() string { + return string(unit) +} + +// IsValid - check whether this validity unit is valid or not. +func (unit ValidityUnit) isValid() bool { + return unit == Days || unit == Years +} + +// Retention - bucket level retention configuration. +type Retention struct { + Mode RetentionMode + Validity time.Duration +} + +func (r Retention) String() string { + return fmt.Sprintf("{Mode:%v, Validity:%v}", r.Mode, r.Validity) +} + +// IsEmpty - returns whether retention is empty or not. +func (r Retention) IsEmpty() bool { + return r.Mode == "" || r.Validity == 0 +} + +// objectLockConfig - object lock configuration specified in +// https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html +type objectLockConfig struct { + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"ObjectLockConfiguration"` + ObjectLockEnabled string `xml:"ObjectLockEnabled"` + Rule *struct { + DefaultRetention struct { + Mode RetentionMode `xml:"Mode"` + Days *uint `xml:"Days"` + Years *uint `xml:"Years"` + } `xml:"DefaultRetention"` + } `xml:"Rule,omitempty"` +} + +func newObjectLockConfig(mode *RetentionMode, validity *uint, unit *ValidityUnit) (*objectLockConfig, error) { + config := &objectLockConfig{ + ObjectLockEnabled: "Enabled", + } + + if mode != nil && validity != nil && unit != nil { + if !mode.IsValid() { + return nil, fmt.Errorf("invalid retention mode `%v`", mode) + } + + if !unit.isValid() { + return nil, fmt.Errorf("invalid validity unit `%v`", unit) + } + + config.Rule = &struct { + DefaultRetention struct { + Mode RetentionMode `xml:"Mode"` + Days *uint `xml:"Days"` + Years *uint `xml:"Years"` + } `xml:"DefaultRetention"` + }{} + + config.Rule.DefaultRetention.Mode = *mode + if *unit == Days { + config.Rule.DefaultRetention.Days = validity + } else { + config.Rule.DefaultRetention.Years = validity + } + + return config, nil + } + + if mode == nil && validity == nil && unit == nil { + return config, nil + } + + return nil, fmt.Errorf("all of retention mode, validity and validity unit must be passed") +} + +// SetBucketObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. +func (c Client) SetBucketObjectLockConfig(bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("object-lock", "") + + config, err := newObjectLockConfig(mode, validity, unit) + if err != nil { + return err + } + + configData, err := xml.Marshal(config) + if err != nil { + return err + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(configData), + contentLength: int64(len(configData)), + contentMD5Base64: sumMD5Base64(configData), + contentSHA256Hex: sum256Hex(configData), + } + + // Execute PUT bucket object lock configuration. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// GetBucketObjectLockConfig gets object lock configuration of given bucket. +func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, nil, nil, err + } + + urlValues := make(url.Values) + urlValues.Set("object-lock", "") + + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return nil, nil, nil, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return nil, nil, nil, httpRespToErrorResponse(resp, bucketName, "") + } + } + config := &objectLockConfig{} + if err = xml.NewDecoder(resp.Body).Decode(config); err != nil { + return nil, nil, nil, err + } + + if config.Rule != nil { + mode = &config.Rule.DefaultRetention.Mode + if config.Rule.DefaultRetention.Days != nil { + validity = config.Rule.DefaultRetention.Days + days := Days + unit = &days + } else { + validity = config.Rule.DefaultRetention.Years + years := Years + unit = &years + } + + return mode, validity, unit, nil + } + + return nil, nil, nil, nil +} diff --git a/api-object-retention.go b/api-object-retention.go new file mode 100644 index 0000000..8aa08f0 --- /dev/null +++ b/api-object-retention.go @@ -0,0 +1,168 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/v6/pkg/s3utils" +) + +// objectRetention - object retention specified in +// https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html +type objectRetention struct { + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"Retention"` + Mode RetentionMode `xml:"Mode"` + RetainUntilDate time.Time `type:"timestamp" timestampFormat:"iso8601" xml:"RetainUntilDate"` +} + +func newObjectRetention(mode *RetentionMode, date *time.Time) (*objectRetention, error) { + if mode == nil { + return nil, fmt.Errorf("Mode not set") + } + + if date == nil { + return nil, fmt.Errorf("RetainUntilDate not set") + } + + if !mode.IsValid() { + return nil, fmt.Errorf("invalid retention mode `%v`", mode) + } + objectRetention := &objectRetention{ + Mode: *mode, + RetainUntilDate: *date, + } + return objectRetention, nil +} + +// PutObjectRetentionOptions represents options specified by user for PutObject call +type PutObjectRetentionOptions struct { + GovernanceBypass bool + Mode *RetentionMode + RetainUntilDate *time.Time + VersionID string +} + +// PutObjectRetention : sets object retention for a given object and versionID. +func (c Client) PutObjectRetention(bucketName, objectName string, opts PutObjectRetentionOptions) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("retention", "") + + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + + retention, err := newObjectRetention(opts.Mode, opts.RetainUntilDate) + if err != nil { + return err + } + + retentionData, err := xml.Marshal(retention) + if err != nil { + return err + } + + // Build headers. + headers := make(http.Header) + + if opts.GovernanceBypass { + // Set the bypass goverenance retention header + headers.Set("x-amz-bypass-governance-retention", "True") + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentBody: bytes.NewReader(retentionData), + contentLength: int64(len(retentionData)), + contentMD5Base64: sumMD5Base64(retentionData), + contentSHA256Hex: sum256Hex(retentionData), + customHeader: headers, + } + + // Execute PUT Object Retention. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, objectName) + } + } + return nil +} + +// GetObjectRetention gets retention of given object. +func (c Client) GetObjectRetention(bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, nil, err + } + + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return nil, nil, err + } + urlValues := make(url.Values) + urlValues.Set("retention", "") + if versionID != "" { + urlValues.Set("versionId", versionID) + } + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return nil, nil, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return nil, nil, httpRespToErrorResponse(resp, bucketName, objectName) + } + } + retention := &objectRetention{} + if err = xml.NewDecoder(resp.Body).Decode(retention); err != nil { + return nil, nil, err + } + + return &retention.Mode, &retention.RetainUntilDate, nil +} diff --git a/api-presigned.go b/api-presigned.go index a2c0607..e2d68b0 100644 --- a/api-presigned.go +++ b/api-presigned.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,8 +23,8 @@ import ( "net/url" "time" - "github.com/minio/minio-go/pkg/s3signer" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3signer" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // presignURL - Returns a presigned URL for an input 'method'. diff --git a/api-put-bucket.go b/api-put-bucket.go index 33dc0cf..0041ce1 100644 --- a/api-put-bucket.go +++ b/api-put-bucket.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,19 +26,12 @@ import ( "net/url" "strings" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) /// Bucket operations -// MakeBucket creates a new bucket with bucketName. -// -// Location is an optional argument, by default all buckets are -// created in US Standard Region. -// -// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html -// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations -func (c Client) MakeBucket(bucketName string, location string) (err error) { +func (c Client) makeBucket(bucketName string, location string, objectLockEnabled bool) (err error) { defer func() { // Save the location into cache on a successful makeBucket response. if err == nil { @@ -66,6 +59,12 @@ func (c Client) MakeBucket(bucketName string, location string) (err error) { bucketLocation: location, } + if objectLockEnabled { + headers := make(http.Header) + headers.Add("x-amz-bucket-object-lock-enabled", "true") + reqMetadata.customHeader = headers + } + // If location is not 'us-east-1' create bucket location config. if location != "us-east-1" && location != "" { createBucketConfig := createBucketConfiguration{} @@ -98,6 +97,28 @@ func (c Client) MakeBucket(bucketName string, location string) (err error) { return nil } +// MakeBucket creates a new bucket with bucketName. +// +// Location is an optional argument, by default all buckets are +// created in US Standard Region. +// +// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html +// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations +func (c Client) MakeBucket(bucketName string, location string) (err error) { + return c.makeBucket(bucketName, location, false) +} + +// MakeBucketWithObjectLock creates a object lock enabled new bucket with bucketName. +// +// Location is an optional argument, by default all buckets are +// created in US Standard Region. +// +// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html +// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations +func (c Client) MakeBucketWithObjectLock(bucketName string, location string) (err error) { + return c.makeBucket(bucketName, location, true) +} + // SetBucketPolicy set the access permissions on an existing bucket. func (c Client) SetBucketPolicy(bucketName, policy string) error { // Input validation. @@ -304,3 +325,59 @@ func (c Client) SetBucketNotification(bucketName string, bucketNotification Buck func (c Client) RemoveAllBucketNotification(bucketName string) error { return c.SetBucketNotification(bucketName, BucketNotification{}) } + +var ( + versionEnableConfig = []byte("<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>Enabled</Status></VersioningConfiguration>") + versionEnableConfigLen = int64(len(versionEnableConfig)) + versionEnableConfigMD5Sum = sumMD5Base64(versionEnableConfig) + versionEnableConfigSHA256 = sum256Hex(versionEnableConfig) + + versionDisableConfig = []byte("<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>Suspended</Status></VersioningConfiguration>") + versionDisableConfigLen = int64(len(versionDisableConfig)) + versionDisableConfigMD5Sum = sumMD5Base64(versionDisableConfig) + versionDisableConfigSHA256 = sum256Hex(versionDisableConfig) +) + +func (c Client) setVersioning(bucketName string, config []byte, length int64, md5sum, sha256sum string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("versioning", "") + + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(config), + contentLength: length, + contentMD5Base64: md5sum, + contentSHA256Hex: sha256sum, + } + + // Execute PUT to set a bucket versioning. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// EnableVersioning - Enable object versioning in given bucket. +func (c Client) EnableVersioning(bucketName string) error { + return c.setVersioning(bucketName, versionEnableConfig, versionEnableConfigLen, versionEnableConfigMD5Sum, versionEnableConfigSHA256) +} + +// DisableVersioning - Disable object versioning in given bucket. +func (c Client) DisableVersioning(bucketName string) error { + return c.setVersioning(bucketName, versionDisableConfig, versionDisableConfigLen, versionDisableConfigMD5Sum, versionDisableConfigSHA256) +} diff --git a/api-put-object-common.go b/api-put-object-common.go index c16c3c6..a786d2a 100644 --- a/api-put-object-common.go +++ b/api-put-object-common.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "math" "os" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // Verify if reader is *minio.Object @@ -34,26 +34,25 @@ func isObject(reader io.Reader) (ok bool) { // Verify if reader is a generic ReaderAt func isReadAt(reader io.Reader) (ok bool) { - _, ok = reader.(io.ReaderAt) + var v *os.File + v, ok = reader.(*os.File) if ok { - var v *os.File - v, ok = reader.(*os.File) - if ok { - // Stdin, Stdout and Stderr all have *os.File type - // which happen to also be io.ReaderAt compatible - // we need to add special conditions for them to - // be ignored by this function. - for _, f := range []string{ - "/dev/stdin", - "/dev/stdout", - "/dev/stderr", - } { - if f == v.Name() { - ok = false - break - } + // Stdin, Stdout and Stderr all have *os.File type + // which happen to also be io.ReaderAt compatible + // we need to add special conditions for them to + // be ignored by this function. + for _, f := range []string{ + "/dev/stdin", + "/dev/stdout", + "/dev/stderr", + } { + if f == v.Name() { + ok = false + break } } + } else { + _, ok = reader.(io.ReaderAt) } return } @@ -65,23 +64,51 @@ func isReadAt(reader io.Reader) (ok bool) { // object storage it will have the following parameters as constants. // // maxPartsCount - 10000 -// minPartSize - 64MiB +// minPartSize - 128MiB // maxMultipartPutObjectSize - 5TiB // -func optimalPartInfo(objectSize int64) (totalPartsCount int, partSize int64, lastPartSize int64, err error) { +func optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize int64, lastPartSize int64, err error) { // object size is '-1' set it to 5TiB. if objectSize == -1 { objectSize = maxMultipartPutObjectSize } + // object size is larger than supported maximum. if objectSize > maxMultipartPutObjectSize { err = ErrEntityTooLarge(objectSize, maxMultipartPutObjectSize, "", "") return } - // Use floats for part size for all calculations to avoid - // overflows during float64 to int64 conversions. - partSizeFlt := math.Ceil(float64(objectSize / maxPartsCount)) - partSizeFlt = math.Ceil(partSizeFlt/minPartSize) * minPartSize + + var partSizeFlt float64 + if configuredPartSize > 0 { + if int64(configuredPartSize) > objectSize { + err = ErrEntityTooLarge(int64(configuredPartSize), objectSize, "", "") + return + } + + if objectSize > (int64(configuredPartSize) * maxPartsCount) { + err = ErrInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") + return + } + + if configuredPartSize < absMinPartSize { + err = ErrInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.") + return + } + + if configuredPartSize > maxPartSize { + err = ErrInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.") + return + } + partSizeFlt = float64(configuredPartSize) + } else { + configuredPartSize = minPartSize + // Use floats for part size for all calculations to avoid + // overflows during float64 to int64 conversions. + partSizeFlt = float64(objectSize / maxPartsCount) + partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize) + } + // Total parts count. totalPartsCount = int(math.Ceil(float64(objectSize) / partSizeFlt)) // Part size. diff --git a/api-put-object-context.go b/api-put-object-context.go index ff4663e..415a787 100644 --- a/api-put-object-context.go +++ b/api-put-object-context.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-put-object-copy.go b/api-put-object-copy.go index 21322ef..19e58ad 100644 --- a/api-put-object-copy.go +++ b/api-put-object-copy.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017, 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017, 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "io/ioutil" "net/http" - "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/encrypt" ) // CopyObject - copy a source object into a new object diff --git a/api-put-object-file-context.go b/api-put-object-file-context.go index 140a9c0..fb22c0d 100644 --- a/api-put-object-file-context.go +++ b/api-put-object-file-context.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "os" "path/filepath" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // FPutObjectWithContext - Create an object in a bucket, with contents from file at filePath. Allows request cancellation. diff --git a/api-put-object-file.go b/api-put-object-file.go index 7c8e051..23df6cd 100644 --- a/api-put-object-file.go +++ b/api-put-object-file.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go index db92520..ab284f9 100644 --- a/api-put-object-multipart.go +++ b/api-put-object-multipart.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,8 +33,8 @@ import ( "strconv" "strings" - "github.com/minio/minio-go/pkg/encrypt" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/s3utils" ) func (c Client) putObjectMultipart(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, @@ -73,7 +73,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje var complMultipartUpload completeMultipartUpload // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, _, err := optimalPartInfo(-1) + totalPartsCount, partSize, _, err := optimalPartInfo(-1, opts.PartSize) if err != nil { return 0, err } diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go index 211d1c2..0d3f245 100644 --- a/api-put-object-streaming.go +++ b/api-put-object-streaming.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import ( "sort" "strings" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // putObjectMultipartStream - upload a large object using @@ -75,7 +75,7 @@ type uploadPartReq struct { Part *ObjectPart // Size of the part uploaded. } -// putObjectMultipartFromReadAt - Uploads files bigger than 64MiB. +// putObjectMultipartFromReadAt - Uploads files bigger than 128MiB. // Supports all readers which implements io.ReaderAt interface // (ReadAt method). // @@ -97,7 +97,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa } // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size) + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size, opts.PartSize) if err != nil { return 0, err } @@ -240,7 +240,7 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa } // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size) + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size, opts.PartSize) if err != nil { return 0, err } diff --git a/api-put-object.go b/api-put-object.go index 0330cd9..bc0848b 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,9 +25,10 @@ import ( "net/http" "runtime/debug" "sort" + "time" - "github.com/minio/minio-go/pkg/encrypt" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/s3utils" "golang.org/x/net/http/httpguts" ) @@ -40,10 +41,13 @@ type PutObjectOptions struct { ContentDisposition string ContentLanguage string CacheControl string + Mode *RetentionMode + RetainUntilDate *time.Time ServerSideEncryption encrypt.ServerSide NumThreads uint StorageClass string WebsiteRedirectLocation string + PartSize uint64 } // getNumThreads - gets the number of threads to be used in the multipart @@ -79,6 +83,14 @@ func (opts PutObjectOptions) Header() (header http.Header) { if opts.CacheControl != "" { header["Cache-Control"] = []string{opts.CacheControl} } + + if opts.Mode != nil { + header["x-amz-object-lock-mode"] = []string{opts.Mode.String()} + } + if opts.RetainUntilDate != nil { + header["x-amz-object-lock-retain-until-date"] = []string{opts.RetainUntilDate.Format(time.RFC3339)} + } + if opts.ServerSideEncryption != nil { opts.ServerSideEncryption.Marshal(header) } @@ -108,6 +120,11 @@ func (opts PutObjectOptions) validate() (err error) { return ErrInvalidArgument(v + " unsupported user defined metadata value") } } + if opts.Mode != nil { + if !opts.Mode.IsValid() { + return ErrInvalidArgument(opts.Mode.String() + " unsupported retention mode") + } + } return nil } @@ -123,9 +140,9 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part // // You must have WRITE permissions on a bucket to create an object. // -// - For size smaller than 64MiB PutObject automatically does a +// - For size smaller than 128MiB PutObject automatically does a // single atomic Put operation. -// - For size larger than 64MiB PutObject automatically does a +// - For size larger than 128MiB PutObject automatically does a // multipart Put operation. // - For size input as -1 PutObject does a multipart Put operation // until input stream reaches EOF. Maximum object size that can @@ -147,8 +164,13 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) } + partSize := opts.PartSize + if opts.PartSize == 0 { + partSize = minPartSize + } + if c.overrideSignerType.IsV2() { - if size >= 0 && size < minPartSize { + if size >= 0 && size < int64(partSize) { return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) } return c.putObjectMultipart(ctx, bucketName, objectName, reader, size, opts) @@ -157,10 +179,11 @@ func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName stri return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts) } - if size < minPartSize { + if size < int64(partSize) { return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) } - // For all sizes greater than 64MiB do multipart. + + // For all sizes greater than 128MiB do multipart. return c.putObjectMultipartStream(ctx, bucketName, objectName, reader, size, opts) } @@ -181,7 +204,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName var complMultipartUpload completeMultipartUpload // Calculate the optimal parts info for a given size. - totalPartsCount, partSize, _, err := optimalPartInfo(-1) + totalPartsCount, partSize, _, err := optimalPartInfo(-1, opts.PartSize) if err != nil { return 0, err } diff --git a/api-put-object_test.go b/api-put-object_test.go index d96abab..2867f3a 100644 --- a/api-put-object_test.go +++ b/api-put-object_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-remove.go b/api-remove.go index f33df4d..4c8c335 100644 --- a/api-remove.go +++ b/api-remove.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import ( "net/http" "net/url" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // RemoveBucket deletes the bucket name. @@ -60,6 +60,17 @@ func (c Client) RemoveBucket(bucketName string) error { // RemoveObject remove an object from a bucket. func (c Client) RemoveObject(bucketName, objectName string) error { + return c.RemoveObjectWithOptions(bucketName, objectName, RemoveObjectOptions{}) +} + +// RemoveObjectOptions represents options specified by user for PutObject call +type RemoveObjectOptions struct { + GovernanceBypass bool + VersionID string +} + +// RemoveObjectWithOptions removes an object from a bucket. +func (c Client) RemoveObjectWithOptions(bucketName, objectName string, opts RemoveObjectOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -67,11 +78,29 @@ func (c Client) RemoveObject(bucketName, objectName string) error { if err := s3utils.CheckValidObjectName(objectName); err != nil { return err } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + + // Build headers. + headers := make(http.Header) + + if opts.GovernanceBypass { + // Set the bypass goverenance retention header + headers.Set("x-amz-bypass-governance-retention", "True") + } // Execute DELETE on objectName. resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ bucketName: bucketName, objectName: objectName, contentSHA256Hex: emptySHA256Hex, + queryValues: urlValues, + customHeader: headers, }) defer closeResponse(resp) if err != nil { diff --git a/api-s3-datatypes.go b/api-s3-datatypes.go index 8d8880c..a6b1255 100644 --- a/api-s3-datatypes.go +++ b/api-s3-datatypes.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/api-select.go b/api-select.go index a9b6f17..7ecd953 100644 --- a/api-select.go +++ b/api-select.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * (C) 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,8 +31,8 @@ import ( "net/url" "strings" - "github.com/minio/minio-go/pkg/encrypt" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // CSVFileHeaderInfo - is the parameter for whether to utilize headers. @@ -90,19 +90,19 @@ type ParquetInputOptions struct{} type CSVInputOptions struct { FileHeaderInfo CSVFileHeaderInfo RecordDelimiter string - FieldDelimiter string - QuoteCharacter string - QuoteEscapeCharacter string - Comments string + FieldDelimiter string `xml:",omitempty"` + QuoteCharacter string `xml:",omitempty"` + QuoteEscapeCharacter string `xml:",omitempty"` + Comments string `xml:",omitempty"` } // CSVOutputOptions csv output specific options type CSVOutputOptions struct { - QuoteFields CSVQuoteFields + QuoteFields CSVQuoteFields `xml:",omitempty"` RecordDelimiter string - FieldDelimiter string - QuoteCharacter string - QuoteEscapeCharacter string + FieldDelimiter string `xml:",omitempty"` + QuoteCharacter string `xml:",omitempty"` + QuoteEscapeCharacter string `xml:",omitempty"` } // JSONInputOptions json input specific options @@ -191,13 +191,20 @@ type StatsMessage struct { BytesReturned int64 } +// messageType represents the type of message. +type messageType string + +const ( + errorMsg messageType = "error" + commonMsg = "event" +) + // eventType represents the type of event. type eventType string // list of event-types returned by Select API. const ( endEvent eventType = "End" - errorEvent = "Error" recordsEvent = "Records" progressEvent = "Progress" statsEvent = "Stats" @@ -244,6 +251,12 @@ func (c Client) SelectObjectContent(ctx context.Context, bucketName, objectName return nil, err } + return NewSelectResults(resp, bucketName) +} + +// NewSelectResults creates a Select Result parser that parses the response +// and returns a Reader that will return parsed and assembled select output. +func NewSelectResults(resp *http.Response, bucketName string) (*SelectResults, error) { if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucketName, "") } @@ -314,53 +327,58 @@ func (s *SelectResults) start(pipeWriter *io.PipeWriter) { // bytes can be read or parsed. payloadLen := prelude.PayloadLen() - // Get content-type of the payload. - c := contentType(headers.Get("content-type")) + m := messageType(headers.Get("message-type")) - // Get event type of the payload. - e := eventType(headers.Get("event-type")) - - // Handle all supported events. - switch e { - case endEvent: - pipeWriter.Close() - closeResponse(s.resp) - return - case errorEvent: - pipeWriter.CloseWithError(errors.New("Error Type of " + headers.Get("error-type") + " " + headers.Get("error-message"))) + switch m { + case errorMsg: + pipeWriter.CloseWithError(errors.New(headers.Get("error-code") + ":\"" + headers.Get("error-message") + "\"")) closeResponse(s.resp) return - case recordsEvent: - if _, err = io.Copy(pipeWriter, io.LimitReader(crcReader, payloadLen)); err != nil { - pipeWriter.CloseWithError(err) + case commonMsg: + // Get content-type of the payload. + c := contentType(headers.Get("content-type")) + + // Get event type of the payload. + e := eventType(headers.Get("event-type")) + + // Handle all supported events. + switch e { + case endEvent: + pipeWriter.Close() closeResponse(s.resp) return - } - case progressEvent: - switch c { - case xmlContent: - if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.progress); err != nil { + case recordsEvent: + if _, err = io.Copy(pipeWriter, io.LimitReader(crcReader, payloadLen)); err != nil { pipeWriter.CloseWithError(err) closeResponse(s.resp) return } - default: - pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, progressEvent)) - closeResponse(s.resp) - return - } - case statsEvent: - switch c { - case xmlContent: - if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.stats); err != nil { - pipeWriter.CloseWithError(err) + case progressEvent: + switch c { + case xmlContent: + if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.progress); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + default: + pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, progressEvent)) + closeResponse(s.resp) + return + } + case statsEvent: + switch c { + case xmlContent: + if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.stats); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + default: + pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, statsEvent)) closeResponse(s.resp) return } - default: - pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, statsEvent)) - closeResponse(s.resp) - return } } diff --git a/api-stat.go b/api-stat.go index 91e9d39..5cc40e7 100644 --- a/api-stat.go +++ b/api-stat.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // BucketExists verify if bucket exists and you have permission to access it. @@ -84,6 +84,7 @@ func extractObjMetadata(header http.Header) http.Header { "Content-Length", "Last-Modified", "Content-Type", + "Expires", }, defaultFilterKeys...) return filterHeader(header, filterKeys) } @@ -170,6 +171,22 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o contentType = "application/octet-stream" } + expiryStr := resp.Header.Get("Expires") + var expTime time.Time + if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil { + expTime = t.UTC() + } + + metadata := extractObjMetadata(resp.Header) + userMetadata := map[string]string{} + const xamzmeta = "x-amz-meta-" + const xamzmetaLen = len(xamzmeta) + for k, v := range metadata { + if strings.HasPrefix(strings.ToLower(k), xamzmeta) { + userMetadata[k[xamzmetaLen:]] = v[0] + } + } + // Save object metadata info. return ObjectInfo{ ETag: md5sum, @@ -177,9 +194,11 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o Size: size, LastModified: date, ContentType: contentType, + Expires: expTime, // Extract only the relevant header keys describing the object. // following function filters out a list of standard set of keys // which are not part of object metadata. - Metadata: extractObjMetadata(resp.Header), + Metadata: metadata, + UserMetadata: userMetadata, }, nil } @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,6 @@ import ( "bytes" "context" "crypto/md5" - "crypto/sha256" "errors" "fmt" "hash" @@ -30,6 +29,7 @@ import ( "math/rand" "net" "net/http" + "net/http/cookiejar" "net/http/httputil" "net/url" "os" @@ -38,9 +38,13 @@ import ( "sync" "time" - "github.com/minio/minio-go/pkg/credentials" - "github.com/minio/minio-go/pkg/s3signer" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/sha256-simd" + + "golang.org/x/net/publicsuffix" + + "github.com/minio/minio-go/v6/pkg/credentials" + "github.com/minio/minio-go/v6/pkg/s3signer" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // Client implements Amazon S3 compatible methods. @@ -70,8 +74,9 @@ type Client struct { bucketLocCache *bucketLocationCache // Advanced functionality. - isTraceEnabled bool - traceOutput io.Writer + isTraceEnabled bool + traceErrorsOnly bool + traceOutput io.Writer // S3 specific accelerated endpoint. s3AccelerateEndpoint string @@ -99,15 +104,15 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v6.0.11" + libraryVersion = "v6.0.43" ) // User Agent should always following the below style. // Please open an issue to discuss any new changes here. // -// Minio (OS; ARCH) LIB/VER APP/VER +// MinIO (OS; ARCH) LIB/VER APP/VER const ( - libraryUserAgentPrefix = "Minio (" + runtime.GOOS + "; " + runtime.GOARCH + ") " + libraryUserAgentPrefix = "MinIO (" + runtime.GOOS + "; " + runtime.GOARCH + ") " libraryUserAgent = libraryUserAgentPrefix + libraryName + "/" + libraryVersion ) @@ -183,6 +188,12 @@ func NewWithOptions(endpoint string, opts *Options) (*Client, error) { return privateNew(endpoint, opts.Creds, opts.Secure, opts.Region, opts.BucketLookup) } +// EndpointURL returns the URL of the S3 endpoint. +func (c *Client) EndpointURL() *url.URL { + endpoint := *c.endpointURL // copy to prevent callers from modifying internal state + return &endpoint +} + // lockedRandSource provides protected rand source, implements rand.Source interface. type lockedRandSource struct { lk sync.Mutex @@ -260,7 +271,7 @@ func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error { case signerType.IsV2(): return errors.New("signature V2 cannot support redirection") case signerType.IsV4(): - req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region)) + s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region)) } } return nil @@ -273,6 +284,13 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re return nil, err } + // Initialize cookies to preserve server sent cookies if any and replay + // them upon each request. + jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + if err != nil { + return nil, err + } + // instantiate new Client. clnt := new(Client) @@ -285,9 +303,15 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re // Save endpoint URL, user agent for future uses. clnt.endpointURL = endpointURL + transport, err := DefaultTransport(secure) + if err != nil { + return nil, err + } + // Instantiate http client and bucket location cache. clnt.httpClient = &http.Client{ - Transport: DefaultTransport, + Jar: jar, + Transport: transport, CheckRedirect: clnt.redirectHeaders, } @@ -314,10 +338,6 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re func (c *Client) SetAppInfo(appName string, appVersion string) { // if app name and version not set, we do not set a new user agent. if appName != "" && appVersion != "" { - c.appInfo = struct { - appName string - appVersion string - }{} c.appInfo.appName = appName c.appInfo.appVersion = appVersion } @@ -357,10 +377,23 @@ func (c *Client) TraceOn(outputStream io.Writer) { c.isTraceEnabled = true } +// TraceErrorsOnlyOn - same as TraceOn, but only errors will be traced. +func (c *Client) TraceErrorsOnlyOn(outputStream io.Writer) { + c.TraceOn(outputStream) + c.traceErrorsOnly = true +} + +// TraceErrorsOnlyOff - Turns off the errors only tracing and everything will be traced after this call. +// If all tracing needs to be turned off, call TraceOff(). +func (c *Client) TraceErrorsOnlyOff() { + c.traceErrorsOnly = false +} + // TraceOff - disable HTTP tracing. func (c *Client) TraceOff() { // Disable tracing. c.isTraceEnabled = false + c.traceErrorsOnly = false } // SetS3TransferAccelerate - turns s3 accelerated endpoint on or off for all your @@ -500,8 +533,9 @@ func (c Client) do(req *http.Request) (*http.Response, error) { return nil, ErrInvalidArgument(msg) } - // If trace is enabled, dump http request and response. - if c.isTraceEnabled { + // If trace is enabled, dump http request and response, + // except when the traceErrorsOnly enabled and the response's status code is ok + if c.isTraceEnabled && !(c.traceErrorsOnly && resp.StatusCode == http.StatusOK) { err = c.dumpHTTP(req, resp) if err != nil { return nil, err @@ -626,14 +660,30 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque // // Additionally we should only retry if bucketLocation and custom // region is empty. - if metadata.bucketLocation == "" && c.region == "" { - if errResponse.Code == "AuthorizationHeaderMalformed" || errResponse.Code == "InvalidRegion" { + if c.region == "" { + switch errResponse.Code { + case "AuthorizationHeaderMalformed": + fallthrough + case "InvalidRegion": + fallthrough + case "AccessDenied": if metadata.bucketName != "" && errResponse.Region != "" { // Gather Cached location only if bucketName is present. - if _, cachedLocationError := c.bucketLocCache.Get(metadata.bucketName); cachedLocationError != false { + if _, cachedOk := c.bucketLocCache.Get(metadata.bucketName); cachedOk { c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) continue // Retry. } + } else { + // Most probably for ListBuckets() + if errResponse.Region != metadata.bucketLocation { + // Retry if the error + // response has a + // different region + // than the request we + // just made. + metadata.bucketLocation = errResponse.Region + continue // Retry + } } } } @@ -667,13 +717,8 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R // Gather location only if bucketName is present. location, err = c.getBucketLocation(metadata.bucketName) if err != nil { - if ToErrorResponse(err).Code != "AccessDenied" { - return nil, err - } + return nil, err } - // Upon AccessDenied error on fetching bucket location, default - // to possible locations based on endpoint URL. This can usually - // happen when GetBucketLocation() is disabled using IAM policies. } if location == "" { location = getDefaultLocation(*c.endpointURL, c.region) @@ -681,10 +726,14 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R } // Look if target url supports virtual host. - isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, metadata.bucketName) + // We explicitly disallow MakeBucket calls to not use virtual DNS style, + // since the resolution may fail. + isMakeBucket := (metadata.objectName == "" && method == "PUT" && len(metadata.queryValues) == 0) + isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, metadata.bucketName) && !isMakeBucket // Construct a new target URL. - targetURL, err := c.makeTargetURL(metadata.bucketName, metadata.objectName, location, isVirtualHost, metadata.queryValues) + targetURL, err := c.makeTargetURL(metadata.bucketName, metadata.objectName, location, + isVirtualHost, metadata.queryValues) if err != nil { return nil, err } diff --git a/api_unit_test.go b/api_unit_test.go index d7822ab..fbfccdb 100644 --- a/api_unit_test.go +++ b/api_unit_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,33 +18,22 @@ package minio import ( - "net/http" "net/url" "testing" - "github.com/minio/minio-go/pkg/credentials" - "github.com/minio/minio-go/pkg/policy" + "github.com/minio/minio-go/v6/pkg/credentials" + "github.com/minio/minio-go/v6/pkg/policy" ) -type customReader struct{} - -func (c *customReader) Read(p []byte) (n int, err error) { - return 0, nil -} - -func (c *customReader) Size() (n int64) { - return 10 -} - // Tests valid hosts for location. func TestValidBucketLocation(t *testing.T) { s3Hosts := []struct { bucketLocation string endpoint string }{ - {"us-east-1", "s3.amazonaws.com"}, - {"unknown", "s3.amazonaws.com"}, - {"ap-southeast-1", "s3-ap-southeast-1.amazonaws.com"}, + {"us-east-1", "s3.dualstack.us-east-1.amazonaws.com"}, + {"unknown", "s3.dualstack.us-east-1.amazonaws.com"}, + {"ap-southeast-1", "s3.dualstack.ap-southeast-1.amazonaws.com"}, } for _, s3Host := range s3Hosts { endpoint := getS3Endpoint(s3Host.bucketLocation) @@ -65,14 +54,8 @@ func TestErrorResponse(t *testing.T) { t.Fatal("Type conversion failed, we have an empty struct.") } - // Test http response decoding. - var httpResponse *http.Response - // Set empty variables - httpResponse = nil - var bucketName, objectName string - // Should fail with invalid argument. - err = httpRespToErrorResponse(httpResponse, bucketName, objectName) + err = httpRespToErrorResponse(nil, "", "") errResp = ToErrorResponse(err) if errResp.Code != "InvalidArgument" { t.Fatal("Empty response input should return invalid argument.") @@ -116,42 +99,61 @@ func TestBucketPolicyTypes(t *testing.T) { // Tests optimal part size. func TestPartSize(t *testing.T) { - _, _, _, err := optimalPartInfo(5000000000000000000) + _, _, _, err := optimalPartInfo(5000000000000000000, minPartSize) if err == nil { t.Fatal("Error: should fail") } - totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(5497558138880) + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(5243928576, 5*1024*1024) if err != nil { t.Fatal("Error: ", err) } - if totalPartsCount != 9103 { - t.Fatalf("Error: expecting total parts count of 9987: got %v instead", totalPartsCount) + if totalPartsCount != 1001 { + t.Fatalf("Error: expecting total parts count of 1001: got %v instead", totalPartsCount) } - if partSize != 603979776 { - t.Fatalf("Error: expecting part size of 550502400: got %v instead", partSize) + if partSize != 5242880 { + t.Fatalf("Error: expecting part size of 5242880: got %v instead", partSize) } - if lastPartSize != 134217728 { - t.Fatalf("Error: expecting last part size of 241172480: got %v instead", lastPartSize) + if lastPartSize != 1048576 { + t.Fatalf("Error: expecting last part size of 1048576: got %v instead", lastPartSize) } - _, partSize, _, err = optimalPartInfo(5000000000) + totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(5243928576, 0) + if err != nil { + t.Fatal("Error: ", err) + } + if totalPartsCount != 40 { + t.Fatalf("Error: expecting total parts count of 40: got %v instead", totalPartsCount) + } + if partSize != 134217728 { + t.Fatalf("Error: expecting part size of 134217728: got %v instead", partSize) + } + if lastPartSize != 9437184 { + t.Fatalf("Error: expecting last part size of 9437184: got %v instead", lastPartSize) + } + _, partSize, _, err = optimalPartInfo(5000000000, minPartSize) if err != nil { t.Fatal("Error:", err) } if partSize != minPartSize { t.Fatalf("Error: expecting part size of %v: got %v instead", minPartSize, partSize) } - totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(-1) + // if stream and client configured min part size + _, _, _, err = optimalPartInfo(-1, minPartSize) + if err == nil { + t.Fatal("Error:", err) + } + // if stream and using default optimal part size determined by sdk + totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(-1, 0) if err != nil { t.Fatal("Error:", err) } - if totalPartsCount != 9103 { - t.Fatalf("Error: expecting total parts count of 9987: got %v instead", totalPartsCount) + if totalPartsCount != 8192 { + t.Fatalf("Error: expecting total parts count of 8192: got %v instead", totalPartsCount) } - if partSize != 603979776 { - t.Fatalf("Error: expecting part size of 550502400: got %v instead", partSize) + if partSize != 671088640 { + t.Fatalf("Error: expecting part size of 671088640: got %v instead", partSize) } - if lastPartSize != 134217728 { - t.Fatalf("Error: expecting last part size of 241172480: got %v instead", lastPartSize) + if lastPartSize != 671088640 { + t.Fatalf("Error: expecting last part size of 671088640: got %v instead", lastPartSize) } } @@ -176,7 +178,7 @@ func TestMakeTargetURL(t *testing.T) { // Test 4, testing against google storage API {"storage.googleapis.com", true, "mybucket", "", "", nil, url.URL{Host: "mybucket.storage.googleapis.com", Scheme: "https", Path: "/"}, nil}, // Test 5, testing against AWS S3 API - {"s3.amazonaws.com", true, "mybucket", "myobject", "", nil, url.URL{Host: "mybucket.s3.amazonaws.com", Scheme: "https", Path: "/myobject"}, nil}, + {"s3.amazonaws.com", true, "mybucket", "myobject", "", nil, url.URL{Host: "mybucket.s3.dualstack.us-east-1.amazonaws.com", Scheme: "https", Path: "/myobject"}, nil}, // Test 6 {"localhost:9000", false, "mybucket", "myobject", "", nil, url.URL{Host: "localhost:9000", Scheme: "http", Path: "/mybucket/myobject"}, nil}, // Test 7, testing with query @@ -188,7 +190,7 @@ func TestMakeTargetURL(t *testing.T) { } for i, testCase := range testCases { - // Initialize a Minio client + // Initialize a MinIO client c, _ := New(testCase.addr, "foo", "bar", testCase.secure) isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, testCase.bucketName) u, err := c.makeTargetURL(testCase.bucketName, testCase.objectName, testCase.bucketLocation, isVirtualHost, testCase.queryValues) diff --git a/appveyor.yml b/appveyor.yml index 48ea6e7..39a33d8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,28 +9,24 @@ clone_folder: c:\gopath\src\github.com\minio\minio-go # environment variables environment: GOPATH: c:\gopath - GO15VENDOREXPERIMENT: 1 + GO111MODULE: on # scripts that run after cloning repository install: - set PATH=%GOPATH%\bin;c:\go\bin;%PATH% - go version - go env - - go get -u golang.org/x/lint/golint - - go get -u github.com/remyoudompheng/go-misc/deadcode - - go get -u github.com/gordonklaus/ineffassign - - go get -u golang.org/x/crypto/argon2 - - go get -t ./... + - go get golang.org/x/lint/golint + - go get honnef.co/go/tools/cmd/staticcheck # to run your custom scripts instead of automatic MSBuild build_script: - go vet ./... - gofmt -s -l . - - golint -set_exit_status github.com/minio/minio-go... - - deadcode - - ineffassign . - - go test -short -v - - go test -short -race -v + - golint -set_exit_status github.com/minio/minio-go/... + - staticcheck + - go test -short -v ./... + - go test -short -race -v ./... # to disable automatic tests test: off diff --git a/bucket-cache.go b/bucket-cache.go index cac7ad7..7ba6cbb 100644 --- a/bucket-cache.go +++ b/bucket-cache.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,14 +18,15 @@ package minio import ( + "net" "net/http" "net/url" "path" "sync" - "github.com/minio/minio-go/pkg/credentials" - "github.com/minio/minio-go/pkg/s3signer" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/credentials" + "github.com/minio/minio-go/v6/pkg/s3signer" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // bucketLocationCache - Provides simple mechanism to hold bucket @@ -123,8 +124,16 @@ func processBucketLocationResponse(resp *http.Response, bucketName string) (buck // For access denied error, it could be an anonymous // request. Move forward and let the top level callers // succeed if possible based on their policy. - if errResp.Code == "AccessDenied" { - return "us-east-1", nil + switch errResp.Code { + case "AuthorizationHeaderMalformed": + fallthrough + case "InvalidRegion": + fallthrough + case "AccessDenied": + if errResp.Region == "" { + return "us-east-1", nil + } + return errResp.Region, nil } return "", err } @@ -161,7 +170,15 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro urlValues.Set("location", "") // Set get bucket location always as path style. - targetURL := c.endpointURL + targetURL := *c.endpointURL + + // as it works in makeTargetURL method from api.go file + if h, p, err := net.SplitHostPort(targetURL.Host); err == nil { + if targetURL.Scheme == "http" && p == "80" || targetURL.Scheme == "https" && p == "443" { + targetURL.Host = h + } + } + targetURL.Path = path.Join(bucketName, "") + "/" targetURL.RawQuery = urlValues.Encode() diff --git a/bucket-cache_test.go b/bucket-cache_test.go index 507b408..e095a5c 100644 --- a/bucket-cache_test.go +++ b/bucket-cache_test.go @@ -1,6 +1,6 @@ /* * Copyright - * 2015, 2016, 2017 Minio, Inc. + * 2015, 2016, 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,8 +27,8 @@ import ( "reflect" "testing" - "github.com/minio/minio-go/pkg/credentials" - "github.com/minio/minio-go/pkg/s3signer" + "github.com/minio/minio-go/v6/pkg/credentials" + "github.com/minio/minio-go/v6/pkg/s3signer" ) // Test validates `newBucketLocationCache`. @@ -67,19 +67,18 @@ func TestBucketLocationCacheOps(t *testing.T) { func TestGetBucketLocationRequest(t *testing.T) { // Generates expected http request for getBucketLocation. // Used for asserting with the actual request generated. - createExpectedRequest := func(c *Client, bucketName string, req *http.Request) (*http.Request, error) { + createExpectedRequest := func(c *Client, bucketName string) (*http.Request, error) { // Set location query. urlValues := make(url.Values) urlValues.Set("location", "") // Set get bucket location always as path style. - targetURL := c.endpointURL + targetURL := *c.endpointURL targetURL.Path = path.Join(bucketName, "") + "/" targetURL.RawQuery = urlValues.Encode() // Get a new HTTP request for the method. - var err error - req, err = http.NewRequest("GET", targetURL.String(), nil) + req, err := http.NewRequest("GET", targetURL.String(), nil) if err != nil { return nil, err } @@ -156,10 +155,10 @@ func TestGetBucketLocationRequest(t *testing.T) { {"storage.googleapis.com", "my-access-key", "my-secret-key", false}, {"storage.googleapis.com", "", "my-secret-key", false}, - // endpoint custom domain running Minio server. - {"play.minio.io", "", "", false}, - {"play.minio.io", "my-access-key", "my-secret-key", false}, - {"play.minio.io", "my-acess-key", "", false}, + // endpoint custom domain running MinIO server. + {"play.min.io", "", "", false}, + {"play.min.io", "my-access-key", "my-secret-key", false}, + {"play.min.io", "my-acess-key", "", false}, } testCases := []struct { bucketName string @@ -251,8 +250,7 @@ func TestGetBucketLocationRequest(t *testing.T) { // Test passes as expected, but the output values are verified for correctness here. if err == nil && testCase.shouldPass { - expectedReq := &http.Request{} - expectedReq, err = createExpectedRequest(client, testCase.bucketName, expectedReq) + expectedReq, err := createExpectedRequest(client, testCase.bucketName) if err != nil { t.Fatalf("Test %d: Expected request Creation failed", i+1) } diff --git a/bucket-notification.go b/bucket-notification.go index ea303dd..4714ead 100644 --- a/bucket-notification.go +++ b/bucket-notification.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ package minio import ( "encoding/xml" - "github.com/minio/minio-go/pkg/set" + "github.com/minio/minio-go/v6/pkg/set" ) // NotificationEventType is a S3 notification event associated to the bucket notification configuration diff --git a/constants.go b/constants.go index 7377423..ac472a6 100644 --- a/constants.go +++ b/constants.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,9 +23,9 @@ package minio // a part in a multipart upload may not be uploaded. const absMinPartSize = 1024 * 1024 * 5 -// minPartSize - minimum part size 64MiB per object after which +// minPartSize - minimum part size 128MiB per object after which // putObject behaves internally as multipart. -const minPartSize = 1024 * 1024 * 64 +const minPartSize = 1024 * 1024 * 128 // maxPartsCount - maximum number of parts for a single multipart session. const maxPartsCount = 10000 @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,9 +20,10 @@ package minio import ( "context" "io" + "net/http" "strings" - "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/v6/pkg/encrypt" ) // Core - Inherits Client and adds new methods to expose the low level S3 APIs. @@ -52,12 +53,26 @@ func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) // ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses // continuationToken instead of marker to support iteration over the results. func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { - return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys, startAfter) + return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, false, delimiter, maxkeys, startAfter) +} + +// CopyObjectWithContext - copies an object from source object to destination object on server side. +func (c Core) CopyObjectWithContext(ctx context.Context, sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { + return c.copyObjectDo(ctx, sourceBucket, sourceObject, destBucket, destObject, metadata) } // CopyObject - copies an object from source object to destination object on server side. func (c Core) CopyObject(sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { - return c.copyObjectDo(context.Background(), sourceBucket, sourceObject, destBucket, destObject, metadata) + return c.CopyObjectWithContext(context.Background(), sourceBucket, sourceObject, destBucket, destObject, metadata) +} + +// CopyObjectPartWithContext - creates a part in a multipart upload by copying (a +// part of) an existing object. +func (c Core) CopyObjectPartWithContext(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, + partID int, startOffset, length int64, metadata map[string]string) (p CompletePart, err error) { + + return c.copyObjectPartDo(ctx, srcBucket, srcObject, destBucket, destObject, uploadID, + partID, startOffset, length, metadata) } // CopyObjectPart - creates a part in a multipart upload by copying (a @@ -65,12 +80,12 @@ func (c Core) CopyObject(sourceBucket, sourceObject, destBucket, destObject stri func (c Core) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset, length int64, metadata map[string]string) (p CompletePart, err error) { - return c.copyObjectPartDo(context.Background(), srcBucket, srcObject, destBucket, destObject, uploadID, + return c.CopyObjectPartWithContext(context.Background(), srcBucket, srcObject, destBucket, destObject, uploadID, partID, startOffset, length, metadata) } -// PutObject - Upload object. Uploads using single PUT call. -func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) { +// PutObjectWithContext - Upload object. Uploads using single PUT call. +func (c Core) PutObjectWithContext(ctx context.Context, bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) { opts := PutObjectOptions{} m := make(map[string]string) for k, v := range metadata { @@ -84,7 +99,7 @@ func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Ba opts.ContentType = v } else if strings.ToLower(k) == "cache-control" { opts.CacheControl = v - } else if strings.ToLower(k) == strings.ToLower(amzWebsiteRedirectLocation) { + } else if strings.EqualFold(k, amzWebsiteRedirectLocation) { opts.WebsiteRedirectLocation = v } else { m[k] = metadata[k] @@ -92,7 +107,12 @@ func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Ba } opts.UserMetadata = m opts.ServerSideEncryption = sse - return c.putObjectDo(context.Background(), bucket, object, data, md5Base64, sha256Hex, size, opts) + return c.putObjectDo(ctx, bucket, object, data, md5Base64, sha256Hex, size, opts) +} + +// PutObject - Upload object. Uploads using single PUT call. +func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) { + return c.PutObjectWithContext(context.Background(), bucket, object, data, size, md5Base64, sha256Hex, metadata, sse) } // NewMultipartUpload - Initiates new multipart upload and returns the new uploadID. @@ -106,9 +126,14 @@ func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, de return c.listMultipartUploadsQuery(bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads) } +// PutObjectPartWithContext - Upload an object part. +func (c Core) PutObjectPartWithContext(ctx context.Context, bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { + return c.uploadPart(ctx, bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse) +} + // PutObjectPart - Upload an object part. func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { - return c.uploadPart(context.Background(), bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse) + return c.PutObjectPartWithContext(context.Background(), bucket, object, uploadID, partID, data, size, md5Base64, sha256Hex, sse) } // ListObjectParts - List uploaded parts of an incomplete upload.x @@ -116,17 +141,27 @@ func (c Core) ListObjectParts(bucket, object, uploadID string, partNumberMarker return c.listObjectPartsQuery(bucket, object, uploadID, partNumberMarker, maxParts) } -// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. -func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) (string, error) { - res, err := c.completeMultipartUpload(context.Background(), bucket, object, uploadID, completeMultipartUpload{ +// CompleteMultipartUploadWithContext - Concatenate uploaded parts and commit to an object. +func (c Core) CompleteMultipartUploadWithContext(ctx context.Context, bucket, object, uploadID string, parts []CompletePart) (string, error) { + res, err := c.completeMultipartUpload(ctx, bucket, object, uploadID, completeMultipartUpload{ Parts: parts, }) return res.ETag, err } +// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. +func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) (string, error) { + return c.CompleteMultipartUploadWithContext(context.Background(), bucket, object, uploadID, parts) +} + +// AbortMultipartUploadWithContext - Abort an incomplete upload. +func (c Core) AbortMultipartUploadWithContext(ctx context.Context, bucket, object, uploadID string) error { + return c.abortMultipartUpload(ctx, bucket, object, uploadID) +} + // AbortMultipartUpload - Abort an incomplete upload. func (c Core) AbortMultipartUpload(bucket, object, uploadID string) error { - return c.abortMultipartUpload(context.Background(), bucket, object, uploadID) + return c.AbortMultipartUploadWithContext(context.Background(), bucket, object, uploadID) } // GetBucketPolicy - fetches bucket access policy for a given bucket. @@ -139,15 +174,28 @@ func (c Core) PutBucketPolicy(bucket, bucketPolicy string) error { return c.putBucketPolicy(bucket, bucketPolicy) } +// GetObjectWithContext is a lower level API implemented to support reading +// partial objects and also downloading objects with special conditions +// matching etag, modtime etc. +func (c Core) GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { + return c.getObject(ctx, bucketName, objectName, opts) +} + // GetObject is a lower level API implemented to support reading // partial objects and also downloading objects with special conditions // matching etag, modtime etc. -func (c Core) GetObject(bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, error) { - return c.getObject(context.Background(), bucketName, objectName, opts) +func (c Core) GetObject(bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { + return c.GetObjectWithContext(context.Background(), bucketName, objectName, opts) +} + +// StatObjectWithContext is a lower level API implemented to support special +// conditions matching etag, modtime on a request. +func (c Core) StatObjectWithContext(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { + return c.statObject(ctx, bucketName, objectName, opts) } // StatObject is a lower level API implemented to support special // conditions matching etag, modtime on a request. func (c Core) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { - return c.statObject(context.Background(), bucketName, objectName, opts) + return c.StatObjectWithContext(context.Background(), bucketName, objectName, opts) } diff --git a/core_test.go b/core_test.go index 5afdd18..e22d5ae 100644 --- a/core_test.go +++ b/core_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,9 @@ import ( "bytes" "io" "log" + "net/http" "os" + "strconv" "testing" "time" @@ -35,8 +37,6 @@ const ( enableSecurity = "ENABLE_HTTPS" ) -// Minimum part size -const MinPartSize = 1024 * 1024 * 64 const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" const ( letterIdxBits = 6 // 6 bits to represent a letter index @@ -86,7 +86,7 @@ func TestGetObjectCore(t *testing.T) { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -123,7 +123,7 @@ func TestGetObjectCore(t *testing.T) { opts := GetObjectOptions{} opts.SetRange(offset, offset+int64(len(buf1))-1) - reader, objectInfo, err := c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err := c.GetObject(bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -142,7 +142,7 @@ func TestGetObjectCore(t *testing.T) { offset += 512 opts.SetRange(offset, offset+int64(len(buf2))-1) - reader, objectInfo, err = c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err = c.GetObject(bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -161,7 +161,7 @@ func TestGetObjectCore(t *testing.T) { } opts.SetRange(0, int64(len(buf3))) - reader, objectInfo, err = c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err = c.GetObject(bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -182,7 +182,7 @@ func TestGetObjectCore(t *testing.T) { opts = GetObjectOptions{} opts.SetMatchETag("etag") - _, _, err = c.GetObject(bucketName, objectName, opts) + _, _, _, err = c.GetObject(bucketName, objectName, opts) if err == nil { t.Fatal("Unexpected GetObject should fail with mismatching etags") } @@ -192,7 +192,7 @@ func TestGetObjectCore(t *testing.T) { opts = GetObjectOptions{} opts.SetMatchETagExcept("etag") - reader, objectInfo, err = c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err = c.GetObject(bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -212,7 +212,7 @@ func TestGetObjectCore(t *testing.T) { opts = GetObjectOptions{} opts.SetRange(0, 0) - reader, objectInfo, err = c.GetObject(bucketName, objectName, opts) + reader, objectInfo, _, err = c.GetObject(bucketName, objectName, opts) if err != nil { t.Fatal(err) } @@ -227,6 +227,23 @@ func TestGetObjectCore(t *testing.T) { t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) } + opts = GetObjectOptions{} + opts.SetRange(offset, offset+int64(len(buf2))-1) + contentLength := len(buf2) + var header http.Header + _, _, header, err = c.GetObject(bucketName, objectName, opts) + if err != nil { + t.Fatal(err) + } + + contentLengthValue, err := strconv.Atoi(header.Get("Content-Length")) + if err != nil { + t.Fatal("Error: ", err) + } + if contentLength != contentLengthValue { + t.Fatalf("Error: Content Length in response header %v, not equal to set content lenght %v\n", contentLengthValue, contentLength) + } + err = c.RemoveObject(bucketName, objectName) if err != nil { t.Fatal("Error: ", err) @@ -262,7 +279,7 @@ func TestGetObjectContentEncoding(t *testing.T) { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -289,7 +306,7 @@ func TestGetObjectContentEncoding(t *testing.T) { t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) } - rwc, objInfo, err := c.GetObject(bucketName, objectName, GetObjectOptions{}) + rwc, objInfo, _, err := c.GetObject(bucketName, objectName, GetObjectOptions{}) if err != nil { t.Fatalf("Error: %v", err) } @@ -304,6 +321,15 @@ func TestGetObjectContentEncoding(t *testing.T) { if value[0] != "gzip" { t.Fatalf("Unexpected content-encoding found, want gzip, got %v", value) } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } } // Tests get bucket policy core API. @@ -330,7 +356,7 @@ func TestGetBucketPolicy(t *testing.T) { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -393,7 +419,7 @@ func TestCoreCopyObject(t *testing.T) { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -507,7 +533,7 @@ func TestCoreCopyObjectPart(t *testing.T) { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -583,7 +609,7 @@ func TestCoreCopyObjectPart(t *testing.T) { // Now we read the data back getOpts := GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) - r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } @@ -597,7 +623,7 @@ func TestCoreCopyObjectPart(t *testing.T) { } getOpts.SetRange(5*1024*1024, 0) - r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } @@ -652,7 +678,7 @@ func TestCorePutObject(t *testing.T) { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -758,7 +784,7 @@ func TestCoreGetObjectMetadata(t *testing.T) { log.Fatalln(err) } - reader, objInfo, err := core.GetObject(bucketName, "my-objectname", GetObjectOptions{}) + reader, objInfo, _, err := core.GetObject(bucketName, "my-objectname", GetObjectOptions{}) if err != nil { log.Fatalln(err) } @@ -767,4 +793,13 @@ func TestCoreGetObjectMetadata(t *testing.T) { if objInfo.Metadata.Get("X-Amz-Meta-Key-1") != "Val-1" { log.Fatalln("Expected metadata to be available but wasn't") } + + err = core.RemoveObject(bucketName, "my-objectname") + if err != nil { + t.Fatal("Error: ", err) + } + err = core.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } } diff --git a/docs/API.md b/docs/API.md index 5778216..f17487d 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,8 +1,8 @@ -# Minio Go Client API Reference [](https://slack.minio.io) +# MinIO Go Client API Reference [](https://slack.min.io) -## Initialize Minio Client object. +## Initialize MinIO Client object. -## Minio +## MinIO ```go package main @@ -10,7 +10,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { @@ -18,7 +18,7 @@ func main() { ssl := true // Initialize minio client object. - minioClient, err := minio.New("play.minio.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) + minioClient, err := minio.New("play.min.io", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) if err != nil { fmt.Println(err) return @@ -34,7 +34,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { @@ -53,25 +53,28 @@ func main() { | Bucket operations | Object operations | Encrypted Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | | :--- | :--- | :--- | :--- | :--- | :--- | | [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`GetObject`](#GetObject) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | -| [`ListBuckets`](#ListBuckets) | [`PutObject`](#PutObject) | [`PutObject`](#PutObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | -| [`BucketExists`](#BucketExists) | [`CopyObject`](#CopyObject) | [`CopyObject`](#CopyObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | -| [`RemoveBucket`](#RemoveBucket) | [`StatObject`](#StatObject) | [`StatObject`](#StatObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | -| [`ListObjects`](#ListObjects) | [`RemoveObject`](#RemoveObject) | | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | -| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveObjects`](#RemoveObjects) | | | [`ListenBucketNotification`](#ListenBucketNotification) | | -| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | -| | [`FPutObject`](#FPutObject) | [`FPutObject`](#FPutObject) | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | -| | [`FGetObject`](#FGetObject) | [`FGetObject`](#FGetObject) | | | | -| | [`ComposeObject`](#ComposeObject) | [`ComposeObject`](#ComposeObject) | | | | -| | [`NewSourceInfo`](#NewSourceInfo) | [`NewSourceInfo`](#NewSourceInfo) | | | | -| | [`NewDestinationInfo`](#NewDestinationInfo) | [`NewDestinationInfo`](#NewDestinationInfo) | | | | +| [`MakeBucketWithObjectLock`](#MakeBucketWithObjectLock) | [`PutObject`](#PutObject) | [`PutObject`](#PutObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | +| [`ListBuckets`](#ListBuckets) | [`CopyObject`](#CopyObject) | [`CopyObject`](#CopyObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | +| [`BucketExists`](#BucketExists) | [`StatObject`](#StatObject) | [`StatObject`](#StatObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | +| [`RemoveBucket`](#RemoveBucket) | [`RemoveObject`](#RemoveObject) | | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | +| [`ListObjects`](#ListObjects) | [`RemoveObjects`](#RemoveObjects) | | | [`ListenBucketNotification`](#ListenBucketNotification) | | +| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | +| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`FPutObject`](#FPutObject) | [`FPutObject`](#FPutObject) | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | +| | [`FGetObject`](#FGetObject) | [`FGetObject`](#FGetObject) | | [`SetBucketObjectLockConfig`](#SetBucketObjectLockConfig) | | +| | [`ComposeObject`](#ComposeObject) | [`ComposeObject`](#ComposeObject) | | [`GetBucketObjectLockConfig`](#GetBucketObjectLockConfig) | | +| | [`NewSourceInfo`](#NewSourceInfo) | [`NewSourceInfo`](#NewSourceInfo) | | [`EnableVersioning`](#EnableVersioning) | | +| | [`NewDestinationInfo`](#NewDestinationInfo) | [`NewDestinationInfo`](#NewDestinationInfo) | | [`DisableVersioning`](#DisableVersioning) | | | | [`PutObjectWithContext`](#PutObjectWithContext) | [`PutObjectWithContext`](#PutObjectWithContext) | | | | | [`GetObjectWithContext`](#GetObjectWithContext) | [`GetObjectWithContext`](#GetObjectWithContext) | | | | | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | | | | [`FGetObjectWithContext`](#FGetObjectWithContext) | [`FGetObjectWithContext`](#FGetObjectWithContext) | | | | | [`RemoveObjectsWithContext`](#RemoveObjectsWithContext) | | | | +| | [`RemoveObjectWithOptions`](#RemoveObjectWithOptions) | | | | +| | [`PutObjectRetention`](#PutObjectRetention) | | | | +| | [`GetObjectRetention`](#GetObjectRetention) | | | | | | [`SelectObjectContent`](#SelectObjectContent) | | ## 1. Constructor -<a name="Minio"></a> +<a name="MinIO"></a> ### New(endpoint, accessKeyID, secretAccessKey string, ssl bool) (*Client, error) Initializes a new client object. @@ -144,6 +147,39 @@ if err != nil { fmt.Println("Successfully created mybucket.") ``` +<a name="MakeBucketWithObjectLock"></a> +### MakeBucketWithObjectLock(bucketName, location string) error +Creates a new bucket with object lock enabled. + +__Parameters__ + +| Param | Type | Description | +|---|---|---| +|`bucketName` | _string_ | Name of the bucket | +| `location` | _string_ | Region where the bucket is to be created. Default value is us-east-1. Other valid values are listed below. Note: When used with minio server, use the region specified in its config file (defaults to us-east-1).| +| | |us-east-1 | +| | |us-west-1 | +| | |us-west-2 | +| | |eu-west-1 | +| | | eu-central-1| +| | | ap-southeast-1| +| | | ap-northeast-1| +| | | ap-southeast-2| +| | | sa-east-1| + + +__Example__ + + +```go +err = minioClient.MakeBucketWithObjectLock("mybucket", "us-east-1") +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully created mybucket.") +``` + <a name="ListBuckets"></a> ### ListBuckets() ([]BucketInfo, error) Lists all buckets. @@ -393,7 +429,7 @@ __minio.GetObjectOptions__ |Field | Type | Description | |:---|:---|:---| -| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | +| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | __Return Value__ @@ -528,7 +564,7 @@ if err != nil { <a name="PutObject"></a> ### PutObject(bucketName, objectName string, reader io.Reader, objectSize int64,opts PutObjectOptions) (n int, err error) -Uploads objects that are less than 64MiB in a single PUT operation. For objects that are greater than 64MiB in size, PutObject seamlessly uploads the object as parts of 64MiB or more depending on the actual file size. The max upload size for an object is 5TB. +Uploads objects that are less than 128MiB in a single PUT operation. For objects that are greater than 128MiB in size, PutObject seamlessly uploads the object as parts of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. __Parameters__ @@ -552,8 +588,10 @@ __minio.PutObjectOptions__ | `opts.ContentDisposition` | _string_ | Content disposition of object, "inline" | | `opts.ContentLanguage` | _string_ | Content language of object, e.g "French" | | `opts.CacheControl` | _string_ | Used to specify directives for caching mechanisms in both requests and responses e.g "max-age=600"| -| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | -| `opts.StorageClass` | _string_ | Specify storage class for the object. Supported values for Minio server are `REDUCED_REDUNDANCY` and `STANDARD` | +| `opts.Mode` | _*minio.RetentionMode_ | Retention mode to be set, e.g "COMPLIANCE" | +| `opts.RetainUntilDate` | _*time.Time_ | Time until which the retention applied is valid| +| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | +| `opts.StorageClass` | _string_ | Specify storage class for the object. Supported values for MinIO server are `REDUCED_REDUNDANCY` and `STANDARD` | | `opts.WebsiteRedirectLocation` | _string_ | Specify a redirect for the object, to another object in the same bucket or to a external URL. | __Example__ @@ -771,7 +809,7 @@ __Parameters__ | :--- | :--- | :--- | | `bucket` | _string_ | Name of the source bucket | | `object` | _string_ | Name of the source object | -| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | +| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | __Example__ @@ -824,7 +862,7 @@ __Parameters__ | :--- | :--- | :--- | | `bucket` | _string_ | Name of the destination bucket | | `object` | _string_ | Name of the destination object | -| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | | +| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | | | `userMeta` | _map[string]string_ | User metadata to be set on the destination. If nil, with only one source, user-metadata is copied from source. | __Example__ @@ -869,7 +907,7 @@ if err != nil { ### FPutObject(bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) Uploads contents from a file to objectName. -FPutObject uploads objects that are less than 64MiB in a single PUT operation. For objects that are greater than the 64MiB in size, FPutObject seamlessly uploads the object in chunks of 64MiB or more depending on the actual file size. The max upload size for an object is 5TB. +FPutObject uploads objects that are less than 128MiB in a single PUT operation. For objects that are greater than the 128MiB in size, FPutObject seamlessly uploads the object in chunks of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. __Parameters__ @@ -1069,6 +1107,94 @@ for rErr := range minioClient.RemoveObjects(ctx, "my-bucketname", objectsCh) { fmt.Println("Error detected during deletion: ", rErr) } ``` +<a name="RemoveObjectWithOptions"></a> +### RemoveObjectWithOptions(bucketName, objectName string, opts minio.RemoveObjectOptions) error +Removes an object. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`opts` |_minio.RemoveObjectOptions_ |Allows user to set options | + +__minio.RemoveObjectOptions__ + +|Field | Type | Description | +|:--- |:--- | :--- | +| `opts.GovernanceBypass` | _bool_ |Set the bypass governance header to delete an object locked with GOVERNANCE mode| +| `opts.VersionID` | _string_ |Version ID of the object to delete| + + +```go +opts := minio.RemoveObjectOptions { + GovernanceBypass: true, + VersionID: "myversionid", + } +err = minioClient.RemoveObjectWithOptions("mybucket", "myobject", opts) +if err != nil { + fmt.Println(err) + return +} +``` +<a name="PutObjectRetention"></a> +### PutObjectRetention(bucketName, objectName string, opts minio.PutObjectRetentionOptions) error +Applies object retention lock onto an object. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`opts` |_minio.PutObjectRetentionOptions_ |Allows user to set options like retention mode, expiry date and version id | + +__minio.PutObjectRetentionOptions__ + +|Field | Type | Description | +|:--- |:--- | :--- | +| `opts.GovernanceBypass` | _bool_ |Set the bypass governance header to overwrite object retention if the existing retention mode is set to GOVERNANCE| +| `opts.Mode` | _*minio.RetentionMode_ |Retention mode to be set| +| `opts.RetainUntilDate` | _*time.Time_ |Time until which the retention applied is valid| +| `opts.VersionID` | _string_ |Version ID of the object to apply retention on| + +```go +t := time.Date(2020, time.November, 18, 14, 0, 0, 0, time.UTC) +m := minio.RetentionMode(minio.Compliance) +opts := minio.PutObjectRetentionOptions { + GovernanceBypass: true, + RetainUntilDate: &t, + Mode: &m, + } +err = minioClient.PutObjectRetention("mybucket", "myobject", opts) +if err != nil { + fmt.Println(err) + return +} +``` +<a name="GetObjectRetention"></a> +### GetObjectRetention(bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) +Returns retention set on a given object. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`versionID` |_string_ |Version ID of the object | + +```go +err = minioClient.PutObjectRetention("mybucket", "myobject", "") +if err != nil { + fmt.Println(err) + return +} +``` <a name="SelectObjectContent"></a> ### SelectObjectContent(ctx context.Context, bucketName string, objectsName string, expression string, options SelectObjectOptions) *SelectResults Parameters @@ -1565,6 +1691,133 @@ if err != nil { } ``` +<a name="SetBucketObjectLockConfig"></a> +### SetBucketObjectLockConfig(bucketname, mode *RetentionMode, validity *uint, unit *ValidityUnit) error +Set object lock configuration in given bucket. mode, validity and unit are either all set or all nil. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket| +|`mode` | _RetentionMode_ |Retention mode to be set | +|`validity` | _uint_ |Validity period to be set | +|`unit` | _ValidityUnit_ |Unit of validity period | + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + +```go +mode := Governance +validity := uint(30) +unit := Days + +err = minioClient.SetBucketObjectLockConfig("my-bucketname", &mode, &validity, &unit) +if err != nil { + fmt.Println(err) + return +} +``` + +<a name="GetBucketObjectLockConfig"></a> +### GetBucketObjectLockConfig(bucketName) (*RetentionMode, *uint, *ValidityUnit, error) +Get object lock configuration of given bucket. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`mode` | _RetentionMode_ |Current retention mode | +|`validity` | _uint_ |Current validity period | +|`unit` | _ValidityUnit_ |Unit of validity period | +|`err` | _error_ |Standard Error | + +__Example__ + +```go +mode, validity, unit, err := minioClient.GetObjectLockConfig("my-bucketname") +if err != nil { + log.Fatalln(err) +} + +if mode != nil { + fmt.Printf("%v mode is enabled for %v %v for bucket 'my-bucketname'\n", *mode, *validity, *unit) +} else { + fmt.Println("No mode is enabled for bucket 'my-bucketname'") +} +``` + +<a name="EnableVersioning"></a> +### EnableVersioning(bucketName) error +Enable bucket versioning support. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + +```go +err := minioClient.EnableVersioning("my-bucketname") +if err != nil { + log.Fatalln(err) +} + +fmt.Println("versioning enabled for bucket 'my-bucketname'") +``` + +<a name="DisableVersioning"></a> +### DisableVersioning(bucketName) error +Disable bucket versioning support. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + +```go +err := minioClient.DisableVersioning("my-bucketname") +if err != nil { + log.Fatalln(err) +} + +fmt.Println("versioning disabled for bucket 'my-bucketname'") +``` + ## 7. Client custom settings <a name="SetAppInfo"></a> @@ -1623,8 +1876,3 @@ __Parameters__ | Param | Type | Description | |---|---|---| |`acceleratedEndpoint` | _string_ | Set to new S3 transfer acceleration endpoint.| - - -## 8. Explore Further - -- [Build your own Go Music Player App example](https://docs.minio.io/docs/go-music-player-app) diff --git a/docs/checker.go.template b/docs/checker.go.template deleted file mode 100644 index 2e0f13a..0000000 --- a/docs/checker.go.template +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/minio/minio-go" -) - -func main() { - // Use a secure connection. - ssl := true - - // Initialize minio client object. - minioClient, err := minio.New("play.minio.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) - if err != nil { - fmt.Println(err) - return - } - - {{.Text}} -} diff --git a/docs/validator.go b/docs/validator.go deleted file mode 100644 index 7d5cbaa..0000000 --- a/docs/validator.go +++ /dev/null @@ -1,227 +0,0 @@ -// +build ignore - -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strings" - "text/template" - - "github.com/a8m/mark" - "github.com/gernest/wow" - "github.com/gernest/wow/spin" - "github.com/minio/cli" -) - -func init() { - // Validate go binary. - if _, err := exec.LookPath("go"); err != nil { - panic(err) - } -} - -var globalFlags = []cli.Flag{ - cli.StringFlag{ - Name: "m", - Value: "API.md", - Usage: "Path to markdown api documentation.", - }, - cli.StringFlag{ - Name: "t", - Value: "checker.go.template", - Usage: "Template used for generating the programs.", - }, - cli.IntFlag{ - Name: "skip", - Value: 2, - Usage: "Skip entries before validating the code.", - }, -} - -func runGofmt(path string) (msg string, err error) { - cmdArgs := []string{"-s", "-w", "-l", path} - cmd := exec.Command("gofmt", cmdArgs...) - stdoutStderr, err := cmd.CombinedOutput() - if err != nil { - return "", err - } - return string(stdoutStderr), nil -} - -func runGoImports(path string) (msg string, err error) { - cmdArgs := []string{"-w", path} - cmd := exec.Command("goimports", cmdArgs...) - stdoutStderr, err := cmd.CombinedOutput() - if err != nil { - return string(stdoutStderr), err - } - return string(stdoutStderr), nil -} - -func runGoBuild(path string) (msg string, err error) { - // Go build the path. - cmdArgs := []string{"build", "-o", "/dev/null", path} - cmd := exec.Command("go", cmdArgs...) - stdoutStderr, err := cmd.CombinedOutput() - if err != nil { - return string(stdoutStderr), err - } - return string(stdoutStderr), nil -} - -func validatorAction(ctx *cli.Context) error { - if !ctx.IsSet("m") || !ctx.IsSet("t") { - return nil - } - docPath := ctx.String("m") - var err error - docPath, err = filepath.Abs(docPath) - if err != nil { - return err - } - data, err := ioutil.ReadFile(docPath) - if err != nil { - return err - } - - templatePath := ctx.String("t") - templatePath, err = filepath.Abs(templatePath) - if err != nil { - return err - } - - skipEntries := ctx.Int("skip") - m := mark.New(string(data), &mark.Options{ - Gfm: true, // Github markdown support is enabled by default. - }) - - t, err := template.ParseFiles(templatePath) - if err != nil { - return err - } - - tmpDir, err := ioutil.TempDir("", "md-verifier") - if err != nil { - return err - } - defer os.RemoveAll(tmpDir) - - entryN := 1 - for i := mark.NodeText; i < mark.NodeCheckbox; i++ { - if mark.NodeCode != mark.NodeType(i) { - m.AddRenderFn(mark.NodeType(i), func(node mark.Node) (s string) { - return "" - }) - continue - } - m.AddRenderFn(mark.NodeCode, func(node mark.Node) (s string) { - p, ok := node.(*mark.CodeNode) - if !ok { - return - } - p.Text = strings.NewReplacer("<", "<", ">", ">", """, `"`, "&", "&").Replace(p.Text) - if skipEntries > 0 { - skipEntries-- - return - } - - testFilePath := filepath.Join(tmpDir, "example.go") - w, werr := os.Create(testFilePath) - if werr != nil { - panic(werr) - } - t.Execute(w, p) - w.Sync() - w.Close() - entryN++ - - msg, err := runGofmt(testFilePath) - if err != nil { - fmt.Printf("Failed running gofmt on %s, with (%s):(%s)\n", testFilePath, msg, err) - os.Exit(-1) - } - - msg, err = runGoImports(testFilePath) - if err != nil { - fmt.Printf("Failed running gofmt on %s, with (%s):(%s)\n", testFilePath, msg, err) - os.Exit(-1) - } - - msg, err = runGoBuild(testFilePath) - if err != nil { - fmt.Printf("Failed running gobuild on %s, with (%s):(%s)\n", testFilePath, msg, err) - fmt.Printf("Code with possible issue in %s:\n%s", docPath, p.Text) - fmt.Printf("To test `go build %s`\n", testFilePath) - os.Exit(-1) - } - - // Once successfully built remove the test file - os.Remove(testFilePath) - return - }) - } - - w := wow.New(os.Stdout, spin.Get(spin.Moon), fmt.Sprintf(" Running validation tests in %s", tmpDir)) - - w.Start() - // Render markdown executes our checker on each code blocks. - _ = m.Render() - w.PersistWith(spin.Get(spin.Runner), " Successfully finished tests") - w.Stop() - - return nil -} - -func main() { - app := cli.NewApp() - app.Action = validatorAction - app.HideVersion = true - app.HideHelpCommand = true - app.Usage = "Validates code block sections inside API.md" - app.Author = "Minio.io" - app.Flags = globalFlags - // Help template for validator - app.CustomAppHelpTemplate = `NAME: - {{.Name}} - {{.Usage}} - -USAGE: - {{.Name}} {{if .VisibleFlags}}[FLAGS] {{end}}COMMAND{{if .VisibleFlags}} [COMMAND FLAGS | -h]{{end}} [ARGUMENTS...] - -COMMANDS: - {{range .VisibleCommands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} - {{end}}{{if .VisibleFlags}} -FLAGS: - {{range .VisibleFlags}}{{.}} - {{end}}{{end}} -TEMPLATE: - Validator uses Go's 'text/template' formatting so you need to ensure - your template is formatted correctly, check 'docs/checker.go.template' - -USAGE: - go run docs/validator.go -m docs/API.md -t /tmp/mycode.go.template - -` - app.Run(os.Args) - -} diff --git a/docs/zh_CN/API.md b/docs/zh_CN/API.md index ca80586..1e47897 100644 --- a/docs/zh_CN/API.md +++ b/docs/zh_CN/API.md @@ -1,8 +1,8 @@ -# Minio Go Client API文档 [](https://slack.minio.io) +# MinIO Go Client API文档 [](https://slack.min.io) -## 初使化Minio Client对象。 +## 初使化MinIO Client对象。 -## Minio +## MinIO ```go package main @@ -10,7 +10,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { @@ -18,7 +18,7 @@ func main() { ssl := true // 初使化minio client对象。 - minioClient, err := minio.New("play.minio.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) + minioClient, err := minio.New("play.min.io", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) if err != nil { fmt.Println(err) return @@ -34,7 +34,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { @@ -69,7 +69,7 @@ func main() { | | [`FPutObjectWithContext`](#FPutObjectWithContext) | | | | | | [`FGetObjectWithContext`](#FGetObjectWithContext) | | | | ## 1. 构造函数 -<a name="Minio"></a> +<a name="MinIO"></a> ### New(endpoint, accessKeyID, secretAccessKey string, ssl bool) (*Client, error) 初使化一个新的client对象。 @@ -379,7 +379,7 @@ __minio.GetObjectOptions__ |参数 | 类型 | 描述 | |:---|:---|:---| -| `opts.Materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go) | +| `opts.Materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go/v6) | __返回值__ @@ -524,7 +524,7 @@ __参数__ |`bucketName` | _string_ |存储桶名称 | |`objectName` | _string_ |对象的名称 | |`filePath` | _string_ |下载后保存的路径| -|`materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go) | +|`materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go/v6) | __示例__ @@ -550,7 +550,7 @@ if err != nil { <a name="PutObject"></a> ### PutObject(bucketName, objectName string, reader io.Reader, objectSize int64,opts PutObjectOptions) (n int, err error) -当对象小于64MiB时，直接在一次PUT请求里进行上传。当大于64MiB时，根据文件的实际大小，PutObject会自动地将对象进行拆分成64MiB一块或更大一些进行上传。对象的最大大小是5TB。 +当对象小于128MiB时，直接在一次PUT请求里进行上传。当大于128MiB时，根据文件的实际大小，PutObject会自动地将对象进行拆分成128MiB一块或更大一些进行上传。对象的最大大小是5TB。 __参数__ @@ -573,7 +573,7 @@ __minio.PutObjectOptions__ | `opts.ContentEncoding` | _string_ | 对象的Content encoding，例如"gzip" | | `opts.ContentDisposition` | _string_ | 对象的Content disposition, "inline" | | `opts.CacheControl` | _string_ | 指定针对请求和响应的缓存机制，例如"max-age=600"| -| `opts.EncryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go) | +| `opts.EncryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go/v6) | __示例__ @@ -889,7 +889,7 @@ if err != nil { ### FPutObject(bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) 将filePath对应的文件内容上传到一个对象中。 -当对象小于64MiB时，FPutObject直接在一次PUT请求里进行上传。当大于64MiB时，根据文件的实际大小，FPutObject会自动地将对象进行拆分成64MiB一块或更大一些进行上传。对象的最大大小是5TB。 +当对象小于128MiB时，FPutObject直接在一次PUT请求里进行上传。当大于128MiB时，根据文件的实际大小，FPutObject会自动地将对象进行拆分成128MiB一块或更大一些进行上传。对象的最大大小是5TB。 __参数__ @@ -1176,7 +1176,7 @@ __参数__ |:---|:---| :---| |`bucketName` | _string_ | 存储桶名称 | |`objectName` | _string_ | 对象的名称 | -|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go) | +|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go/v6) | __返回值__ @@ -1233,7 +1233,7 @@ __参数__ |`bucketName` | _string_ |存储桶名称 | |`objectName` | _string_ |对象的名称 | |`reader` | _io.Reader_ |任何实现io.Reader的Go类型 | -|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go) | +|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go/v6) | __示例__ @@ -1295,7 +1295,7 @@ __参数__ |`bucketName` | _string_ |存储桶名称 | |`objectName` | _string_ |对象的名称 | |`filePath` | _string_ |要上传的文件的路径 | -|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go) | +|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go/v6) | __示例__ @@ -1489,7 +1489,7 @@ fmt.Printf("%s\n", url) ### SetBucketPolicy(bucketname, objectPrefix string, policy policy.BucketPolicy) error 给存储桶或者对象前缀设置访问权限。 -必须引入`github.com/minio/minio-go/pkg/policy`包。 +必须引入`github.com/minio/minio-go/v6/pkg/policy`包。 __参数__ @@ -1530,7 +1530,7 @@ if err != nil { ### GetBucketPolicy(bucketName, objectPrefix string) (policy.BucketPolicy, error) 获取存储桶或者对象前缀的访问权限。 -必须引入`github.com/minio/minio-go/pkg/policy`包。 +必须引入`github.com/minio/minio-go/v6/pkg/policy`包。 __参数__ @@ -1779,8 +1779,3 @@ __参数__ | 参数 | 类型 | 描述 | |---|---|---| |`acceleratedEndpoint` | _string_ | 设置新的S3传输加速endpoint。| - - -## 8. 了解更多 - -- [用Go语言创建属于你的音乐播放器APP示例](https://docs.minio.io/docs/go-music-player-app) diff --git a/examples/minio/listenbucketnotification.go b/examples/minio/listenbucketnotification.go index 4c48510..a899c14 100644 --- a/examples/minio/listenbucketnotification.go +++ b/examples/minio/listenbucketnotification.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { @@ -34,7 +34,7 @@ func main() { // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - minioClient, err := minio.New("play.minio.io:9000", "YOUR-ACCESS", "YOUR-SECRET", true) + minioClient, err := minio.New("play.min.io", "YOUR-ACCESS", "YOUR-SECRET", true) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/bucketexists.go b/examples/s3/bucketexists.go index 20dea30..e4b0a3a 100644 --- a/examples/s3/bucketexists.go +++ b/examples/s3/bucketexists.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/composeobject.go b/examples/s3/composeobject.go index 2f76ff0..78bfe95 100644 --- a/examples/s3/composeobject.go +++ b/examples/s3/composeobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,8 @@ package main import ( "log" - minio "github.com/minio/minio-go" + minio "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6/pkg/encrypt" ) func main() { @@ -44,27 +45,27 @@ func main() { // Prepare source decryption key (here we assume same key to // decrypt all source objects.) - decKey := minio.NewSSEInfo([]byte{1, 2, 3}, "") + decKey, _ := encrypt.NewSSEC([]byte{1, 2, 3}) // Source objects to concatenate. We also specify decryption // key for each - src1 := minio.NewSourceInfo("bucket1", "object1", &decKey) + src1 := minio.NewSourceInfo("bucket1", "object1", decKey) src1.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") - src2 := minio.NewSourceInfo("bucket2", "object2", &decKey) + src2 := minio.NewSourceInfo("bucket2", "object2", decKey) src2.SetMatchETagCond("f8ef9c385918b653a31624deb84149d2") - src3 := minio.NewSourceInfo("bucket3", "object3", &decKey) + src3 := minio.NewSourceInfo("bucket3", "object3", decKey) src3.SetMatchETagCond("5918b653a31624deb84149d2f8ef9c38") // Create slice of sources. srcs := []minio.SourceInfo{src1, src2, src3} // Prepare destination encryption key - encKey := minio.NewSSEInfo([]byte{8, 9, 0}, "") + encKey, _ := encrypt.NewSSEC([]byte{8, 9, 0}) // Create destination info - dst, err := minio.NewDestinationInfo("bucket", "object", &encKey, nil) + dst, err := minio.NewDestinationInfo("bucket", "object", encKey, nil) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/copyobject.go b/examples/s3/copyobject.go index a7c3eca..1c3f65a 100644 --- a/examples/s3/copyobject.go +++ b/examples/s3/copyobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "log" "time" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/disableversioning.go b/examples/s3/disableversioning.go new file mode 100644 index 0000000..bbaec5e --- /dev/null +++ b/examples/s3/disableversioning.go @@ -0,0 +1,47 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + minio "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + err = s3Client.DisableVersioning("my-bucketname") + if err != nil { + log.Fatalln(err) + } + log.Println("Disabled") +} diff --git a/examples/s3/enableversioning.go b/examples/s3/enableversioning.go new file mode 100644 index 0000000..58179be --- /dev/null +++ b/examples/s3/enableversioning.go @@ -0,0 +1,47 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + minio "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + err = s3Client.EnableVersioning("my-bucketname") + if err != nil { + log.Fatalln(err) + } + log.Println("Enabled") +} diff --git a/examples/s3/fgetobject-context.go b/examples/s3/fgetobject-context.go index 6004baa..f3dc74d 100644 --- a/examples/s3/fgetobject-context.go +++ b/examples/s3/fgetobject-context.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import ( "context" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/fgetobject.go b/examples/s3/fgetobject.go index 819a34f..76dd59f 100644 --- a/examples/s3/fgetobject.go +++ b/examples/s3/fgetobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/fputencrypted-object.go b/examples/s3/fputencrypted-object.go index 5da9f9d..95eefea 100644 --- a/examples/s3/fputencrypted-object.go +++ b/examples/s3/fputencrypted-object.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,9 +22,8 @@ package main import ( "log" - "github.com/minio/minio-go/pkg/encrypt" - - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6/pkg/encrypt" ) func main() { diff --git a/examples/s3/fputobject-context.go b/examples/s3/fputobject-context.go index d7c941c..0aa300c 100644 --- a/examples/s3/fputobject-context.go +++ b/examples/s3/fputobject-context.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import ( "context" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/fputobject.go b/examples/s3/fputobject.go index 34d8768..2fc3595 100644 --- a/examples/s3/fputobject.go +++ b/examples/s3/fputobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/get-encrypted-object.go b/examples/s3/get-encrypted-object.go index 62a06d5..260770d 100644 --- a/examples/s3/get-encrypted-object.go +++ b/examples/s3/get-encrypted-object.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,8 +24,8 @@ import ( "log" "os" - "github.com/minio/minio-go" - "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6/pkg/encrypt" ) func main() { diff --git a/examples/s3/getbucketlifecycle.go b/examples/s3/getbucketlifecycle.go index 2e3ef41..1982a76 100644 --- a/examples/s3/getbucketlifecycle.go +++ b/examples/s3/getbucketlifecycle.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import ( "os" "strings" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/getbucketnotification.go b/examples/s3/getbucketnotification.go index 19349ba..1d711fc 100644 --- a/examples/s3/getbucketnotification.go +++ b/examples/s3/getbucketnotification.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/getbucketobjectlockconfig.go b/examples/s3/getbucketobjectlockconfig.go new file mode 100644 index 0000000..ddd9f92 --- /dev/null +++ b/examples/s3/getbucketobjectlockconfig.go @@ -0,0 +1,56 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + + minio "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Get object lock configuration. + mode, validity, unit, err := s3Client.GetBucketObjectLockConfig("my-bucketname") + if err != nil { + log.Fatalln(err) + } + + if mode != nil { + fmt.Printf("%v mode is enabled for %v %v for bucket 'my-bucketname'\n", *mode, *validity, *unit) + } else { + fmt.Println("No mode is enabled for bucket 'my-bucketname'") + } +} diff --git a/examples/s3/getbucketpolicy.go b/examples/s3/getbucketpolicy.go index e5b5940..b10e933 100644 --- a/examples/s3/getbucketpolicy.go +++ b/examples/s3/getbucketpolicy.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import ( "os" "strings" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/getobject-client-encryption.go b/examples/s3/getobject-client-encryption.go index 6b06073..33fb0d2 100644 --- a/examples/s3/getobject-client-encryption.go +++ b/examples/s3/getobject-client-encryption.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ import ( "os" "path" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" "github.com/minio/sio" "golang.org/x/crypto/argon2" ) diff --git a/examples/s3/getobject-context.go b/examples/s3/getobject-context.go index c7d4170..1764d8a 100644 --- a/examples/s3/getobject-context.go +++ b/examples/s3/getobject-context.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ import ( "context" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/getobject.go b/examples/s3/getobject.go index e17ef81..eab9fc5 100644 --- a/examples/s3/getobject.go +++ b/examples/s3/getobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ import ( "log" "os" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/getobjectacl.go b/examples/s3/getobjectacl.go index f2bbd95..f557009 100644 --- a/examples/s3/getobjectacl.go +++ b/examples/s3/getobjectacl.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018-2019 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018-2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "fmt" "log" - minio "github.com/minio/minio-go" + minio "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/getobjectretention.go b/examples/s3/getobjectretention.go new file mode 100644 index 0000000..527f8a2 --- /dev/null +++ b/examples/s3/getobjectretention.go @@ -0,0 +1,48 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "io" + "log" + "os" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + m, t, err = s3Client.GetObjectRetention("my-bucket", "my-object", "") + if err != nil { + log.Fatalln(err) + } + log.Println("Get object retention successful, Mode: ", m.String(), " Retainuntil Date ", t.String()) +} diff --git a/examples/s3/listbuckets.go b/examples/s3/listbuckets.go index 5eae587..9b1e23e 100644 --- a/examples/s3/listbuckets.go +++ b/examples/s3/listbuckets.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/listincompleteuploads.go b/examples/s3/listincompleteuploads.go index a5a79b6..da9bf59 100644 --- a/examples/s3/listincompleteuploads.go +++ b/examples/s3/listincompleteuploads.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "fmt" "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/listobjects-N.go b/examples/s3/listobjects-N.go index 55bceb4..12afbbd 100644 --- a/examples/s3/listobjects-N.go +++ b/examples/s3/listobjects-N.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/listobjects.go b/examples/s3/listobjects.go index 1da2e3f..34fed84 100644 --- a/examples/s3/listobjects.go +++ b/examples/s3/listobjects.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/listobjectsV2.go b/examples/s3/listobjectsV2.go index 190aec3..7443e8b 100644 --- a/examples/s3/listobjectsV2.go +++ b/examples/s3/listobjectsV2.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "fmt" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/listobjectsV2WithMetadata.go b/examples/s3/listobjectsV2WithMetadata.go new file mode 100644 index 0000000..77eb509 --- /dev/null +++ b/examples/s3/listobjectsV2WithMetadata.go @@ -0,0 +1,58 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + fmt.Println(err) + return + } + + // Create a done channel to control 'ListObjects' go routine. + doneCh := make(chan struct{}) + + // Indicate to our routine to exit cleanly upon return. + defer close(doneCh) + + // List all objects from a bucket-name with a matching prefix. + for object := range s3Client.ListObjectsV2WithMetadata("my-bucketname", "my-prefixname", true, doneCh) { + if object.Err != nil { + fmt.Println(object.Err) + return + } + fmt.Println(object) + } + return +} diff --git a/examples/s3/makebucket.go b/examples/s3/makebucket.go index 419c96c..1e17995 100644 --- a/examples/s3/makebucket.go +++ b/examples/s3/makebucket.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/makebucketwithobjectlock.go b/examples/s3/makebucketwithobjectlock.go new file mode 100644 index 0000000..af52dd0 --- /dev/null +++ b/examples/s3/makebucketwithobjectlock.go @@ -0,0 +1,47 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + minio "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + err = s3Client.MakeBucketWithObjectLock("my-bucketname", "us-east-1") + if err != nil { + log.Fatalln(err) + } + log.Println("Success") +} diff --git a/examples/s3/presignedgetobject.go b/examples/s3/presignedgetobject.go index fd7fb9e..dc870a0 100644 --- a/examples/s3/presignedgetobject.go +++ b/examples/s3/presignedgetobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ import ( "net/url" "time" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/presignedheadobject.go b/examples/s3/presignedheadobject.go index 8dbc0a4..8cec788 100644 --- a/examples/s3/presignedheadobject.go +++ b/examples/s3/presignedheadobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ import ( "net/url" "time" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/presignedpostpolicy.go b/examples/s3/presignedpostpolicy.go index 205ac95..25e0f74 100644 --- a/examples/s3/presignedpostpolicy.go +++ b/examples/s3/presignedpostpolicy.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ import ( "log" "time" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/presignedputobject.go b/examples/s3/presignedputobject.go index b2f8b4f..d200faa 100644 --- a/examples/s3/presignedputobject.go +++ b/examples/s3/presignedputobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "log" "time" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/put-encrypted-object.go b/examples/s3/put-encrypted-object.go index 48b9367..7d4cde2 100644 --- a/examples/s3/put-encrypted-object.go +++ b/examples/s3/put-encrypted-object.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,8 +23,8 @@ import ( "log" "os" - "github.com/minio/minio-go" - "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6/pkg/encrypt" ) func main() { diff --git a/examples/s3/putobject-client-encryption.go b/examples/s3/putobject-client-encryption.go index 77d83b4..0c0d7bb 100644 --- a/examples/s3/putobject-client-encryption.go +++ b/examples/s3/putobject-client-encryption.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ import ( "os" "path" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" "github.com/minio/sio" "golang.org/x/crypto/argon2" ) diff --git a/examples/s3/putobject-context.go b/examples/s3/putobject-context.go index acc923f..6a476d4 100644 --- a/examples/s3/putobject-context.go +++ b/examples/s3/putobject-context.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ import ( "context" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/putobject-getobject-sse.go b/examples/s3/putobject-getobject-sse.go index 4e459b5..110c323 100644 --- a/examples/s3/putobject-getobject-sse.go +++ b/examples/s3/putobject-getobject-sse.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,9 +24,8 @@ import ( "io/ioutil" "log" - "github.com/minio/minio-go/pkg/encrypt" - - minio "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6/pkg/encrypt" ) func main() { diff --git a/examples/s3/putobject-progress.go b/examples/s3/putobject-progress.go index 0e92dd6..54ce7cb 100644 --- a/examples/s3/putobject-progress.go +++ b/examples/s3/putobject-progress.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "log" "github.com/cheggaaa/pb" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/putobject-s3-accelerate.go b/examples/s3/putobject-s3-accelerate.go index 06345cd..d36af0a 100644 --- a/examples/s3/putobject-s3-accelerate.go +++ b/examples/s3/putobject-s3-accelerate.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "log" "os" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/putobject-streaming.go b/examples/s3/putobject-streaming.go index 85b78dd..a90e472 100644 --- a/examples/s3/putobject-streaming.go +++ b/examples/s3/putobject-streaming.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "log" "os" - minio "github.com/minio/minio-go" + minio "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/putobject.go b/examples/s3/putobject.go index b9e4ff1..31969e0 100644 --- a/examples/s3/putobject.go +++ b/examples/s3/putobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "log" "os" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/putobjectretention.go b/examples/s3/putobjectretention.go new file mode 100644 index 0000000..b0db21a --- /dev/null +++ b/examples/s3/putobjectretention.go @@ -0,0 +1,54 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "time" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + t := time.Date(2020, time.November, 18, 14, 0, 0, 0, time.UTC) + m := minio.RetentionMode(minio.Governance) + opts := minio.PutObjectRetentionOptions{ + GovernanceBypass: true, + RetainUntilDate: &t, + Mode: &m, + } + err = s3Client.PutObjectRetention("my-bucket", "my-object", opts) + if err != nil { + log.Fatalln(err) + } + log.Println("Set object retention on my-object successfully.") +} diff --git a/examples/s3/removeallbucketnotification.go b/examples/s3/removeallbucketnotification.go index 1186afa..afd8ce1 100644 --- a/examples/s3/removeallbucketnotification.go +++ b/examples/s3/removeallbucketnotification.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/removebucket.go b/examples/s3/removebucket.go index 7a7737e..7f2295e 100644 --- a/examples/s3/removebucket.go +++ b/examples/s3/removebucket.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/removeincompleteupload.go b/examples/s3/removeincompleteupload.go index 31cc879..9b941ea 100644 --- a/examples/s3/removeincompleteupload.go +++ b/examples/s3/removeincompleteupload.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/removeobject.go b/examples/s3/removeobject.go index 7e58485..967ec8c 100644 --- a/examples/s3/removeobject.go +++ b/examples/s3/removeobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/removeobjectoptions.go b/examples/s3/removeobjectoptions.go new file mode 100644 index 0000000..926d80e --- /dev/null +++ b/examples/s3/removeobjectoptions.go @@ -0,0 +1,49 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + opts := minio.RemoveObjectOptions{ + GovernanceBypass: true, + } + err = s3Client.RemoveObjectWithOptions("my-bucket", "my-object", opts) + if err != nil { + log.Fatalln(err) + } + log.Println("Remove object successful") +} diff --git a/examples/s3/removeobjects.go b/examples/s3/removeobjects.go index a581134..d453bee 100644 --- a/examples/s3/removeobjects.go +++ b/examples/s3/removeobjects.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/selectobject.go b/examples/s3/selectobject.go index e23ccf8..e7a8370 100644 --- a/examples/s3/selectobject.go +++ b/examples/s3/selectobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import ( "log" "os" - minio "github.com/minio/minio-go" + minio "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/setbucketlifecycle.go b/examples/s3/setbucketlifecycle.go index 7eaa946..362caa8 100644 --- a/examples/s3/setbucketlifecycle.go +++ b/examples/s3/setbucketlifecycle.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/setbucketnotification.go b/examples/s3/setbucketnotification.go index b5af30f..bb6ab5e 100644 --- a/examples/s3/setbucketnotification.go +++ b/examples/s3/setbucketnotification.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/setbucketobjectlockconfig.go b/examples/s3/setbucketobjectlockconfig.go new file mode 100644 index 0000000..8034969 --- /dev/null +++ b/examples/s3/setbucketobjectlockconfig.go @@ -0,0 +1,54 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + minio "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Set object lock configuration. + mode := minio.Governance + validity := uint(30) + unit := minio.Days + + err = s3Client.SetBucketObjectLockConfig("my-bucketname", &mode, &validity, &unit) + if err != nil { + log.Fatalln(err) + } + log.Println("Success") +} diff --git a/examples/s3/setbucketpolicy.go b/examples/s3/setbucketpolicy.go index bc42da8..6076278 100644 --- a/examples/s3/setbucketpolicy.go +++ b/examples/s3/setbucketpolicy.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/examples/s3/statobject.go b/examples/s3/statobject.go index 0b27a83..9e8fec6 100644 --- a/examples/s3/statobject.go +++ b/examples/s3/statobject.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ package main import ( "log" - "github.com/minio/minio-go" + "github.com/minio/minio-go/v6" ) func main() { diff --git a/functional_tests.go b/functional_tests.go index 62be0e9..69ae3b2 100644 --- a/functional_tests.go +++ b/functional_tests.go @@ -1,8 +1,8 @@ // +build ignore /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,10 +40,10 @@ import ( "time" humanize "github.com/dustin/go-humanize" - minio "github.com/minio/minio-go" log "github.com/sirupsen/logrus" - "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v6/pkg/encrypt" ) const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" @@ -57,6 +57,7 @@ const ( accessKey = "ACCESS_KEY" secretKey = "SECRET_KEY" enableHTTPS = "ENABLE_HTTPS" + enableKMS = "ENABLE_KMS" ) type mintJSONFormatter struct { @@ -183,9 +184,9 @@ func isErrNotImplemented(err error) bool { func init() { // If server endpoint is not set, all tests default to - // using https://play.minio.io:9000 + // using https://play.min.io if os.Getenv(serverEndpoint) == "" { - os.Setenv(serverEndpoint, "play.minio.io:9000") + os.Setenv(serverEndpoint, "play.min.io") os.Setenv(accessKey, "Q3AM3UQ867SPQQA43P2F") os.Setenv(secretKey, "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") os.Setenv(enableHTTPS, "1") @@ -263,7 +264,7 @@ var dataFileMap = map[string]int{ "datafile-5-MB": 5 * humanize.MiByte, "datafile-6-MB": 6 * humanize.MiByte, "datafile-11-MB": 11 * humanize.MiByte, - "datafile-65-MB": 65 * humanize.MiByte, + "datafile-129-MB": 129 * humanize.MiByte, } func isFullMode() bool { @@ -306,7 +307,7 @@ func testMakeBucketError() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return } @@ -314,7 +315,7 @@ func testMakeBucketError() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -362,10 +363,10 @@ func testMetadataSizeLimit() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return } - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName @@ -442,7 +443,7 @@ func testMakeBucketRegions() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return } @@ -450,7 +451,7 @@ func testMakeBucketRegions() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -509,7 +510,7 @@ func testPutObjectReadAt() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -517,7 +518,7 @@ func testPutObjectReadAt() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -530,8 +531,8 @@ func testPutObjectReadAt() { return } - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") defer reader.Close() // Save the data @@ -619,7 +620,7 @@ func testPutObjectWithMetadata() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -627,7 +628,7 @@ func testPutObjectWithMetadata() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -640,8 +641,8 @@ func testPutObjectWithMetadata() { return } - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") defer reader.Close() // Save the data @@ -652,7 +653,8 @@ func testPutObjectWithMetadata() { customContentType := "custom/contenttype" args["metadata"] = map[string][]string{ - "Content-Type": {customContentType}, + "Content-Type": {customContentType}, + "X-Amz-Meta-CustomKey": {"extra spaces in value"}, } n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ @@ -729,7 +731,7 @@ func testPutObjectWithContentLanguage() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -737,7 +739,7 @@ func testPutObjectWithContentLanguage() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -808,7 +810,7 @@ func testPutObjectStreaming() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -816,7 +818,7 @@ func testPutObjectStreaming() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -873,7 +875,7 @@ func testGetObjectSeekEnd() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -881,7 +883,7 @@ func testGetObjectSeekEnd() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1005,7 +1007,7 @@ func testGetObjectClosedTwice() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1013,7 +1015,7 @@ func testGetObjectClosedTwice() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1101,12 +1103,12 @@ func testRemoveObjectsWithContext() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Enable tracing, write to stdout. // c.TraceOn(os.Stderr) @@ -1197,12 +1199,12 @@ func testRemoveMultipleObjects() { ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Enable tracing, write to stdout. // c.TraceOn(os.Stderr) @@ -1284,7 +1286,7 @@ func testFPutObjectMultipart() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1292,7 +1294,7 @@ func testFPutObjectMultipart() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1306,7 +1308,7 @@ func testFPutObjectMultipart() { } // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. - var fileName = getMintDataDirFilePath("datafile-65-MB") + var fileName = getMintDataDirFilePath("datafile-129-MB") if fileName == "" { // Make a temp file with minPartSize bytes of data. file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") @@ -1315,7 +1317,7 @@ func testFPutObjectMultipart() { return } // Upload 2 parts to utilize all 3 'workers' in multipart and still have a part to upload. - if _, err = io.Copy(file, getDataReader("datafile-65-MB")); err != nil { + if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } @@ -1326,7 +1328,7 @@ func testFPutObjectMultipart() { fileName = file.Name() args["fileName"] = fileName } - totalSize := dataFileMap["datafile-65-MB"] + totalSize := dataFileMap["datafile-129-MB"] // Set base object name objectName := bucketName + "FPutObject" + "-standard" args["objectName"] = objectName @@ -1398,7 +1400,7 @@ func testFPutObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1406,7 +1408,7 @@ func testFPutObject() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1424,7 +1426,7 @@ func testFPutObject() { // Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. // Use different data in part for multipart tests to check parts are uploaded in correct order. - var fName = getMintDataDirFilePath("datafile-65-MB") + var fName = getMintDataDirFilePath("datafile-129-MB") if fName == "" { // Make a temp file with minPartSize bytes of data. file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") @@ -1434,7 +1436,7 @@ func testFPutObject() { } // Upload 3 parts to utilize all 3 'workers' in multipart and still have a part to upload. - if _, err = io.Copy(file, getDataReader("datafile-65-MB")); err != nil { + if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil { logError(testName, function, args, startTime, "", "File copy failed", err) return } @@ -1446,7 +1448,7 @@ func testFPutObject() { defer os.Remove(file.Name()) fName = file.Name() } - totalSize := dataFileMap["datafile-65-MB"] + totalSize := dataFileMap["datafile-129-MB"] // Set base object name function = "FPutObject(bucketName, objectName, fileName, opts)" @@ -1584,7 +1586,7 @@ func testFPutObjectWithContext() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1592,7 +1594,7 @@ func testFPutObjectWithContext() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1695,7 +1697,7 @@ func testFPutObjectWithContextV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1703,7 +1705,7 @@ func testFPutObjectWithContextV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1806,7 +1808,7 @@ func testPutObjectWithContext() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1814,7 +1816,7 @@ func testPutObjectWithContext() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -1883,7 +1885,7 @@ func testGetObjectReadSeekFunctional() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -1891,7 +1893,7 @@ func testGetObjectReadSeekFunctional() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -2058,7 +2060,7 @@ func testGetObjectReadAtFunctional() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -2066,7 +2068,7 @@ func testGetObjectReadAtFunctional() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -2224,6 +2226,133 @@ func testGetObjectReadAtFunctional() { successLogger(testName, function, args, startTime).Info() } +// Reproduces issue https://github.com/minio/minio-go/issues/1137 +func testGetObjectReadAtWhenEOFWasReached() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + buf, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + // Save the data + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) + return + } + + // read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + // read directly + buf1 := make([]byte, n) + buf2 := make([]byte, 512) + + m, err := r.Read(buf1) + if err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "Read failed", err) + return + } + } + if m != len(buf1) { + logError(testName, function, args, startTime, "", "Read read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) + return + } + if !bytes.Equal(buf1, buf) { + logError(testName, function, args, startTime, "", "Incorrect count of Read data", err) + return + } + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + return + } + + m, err = r.ReadAt(buf2, 512) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf2) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) + return + } + if !bytes.Equal(buf2, buf[512:1024]) { + logError(testName, function, args, startTime, "", "Incorrect count of ReadAt data", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + // Test Presigned Post Policy func testPresignedPostPolicy() { // initialize logging params @@ -2245,7 +2374,7 @@ func testPresignedPostPolicy() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -2253,7 +2382,7 @@ func testPresignedPostPolicy() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -2445,7 +2574,7 @@ func testCopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -2453,7 +2582,7 @@ func testCopyObject() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -2657,7 +2786,7 @@ func testCopyObject() { } // Tests SSE-C get object ReaderSeeker interface methods. -func testEncryptedGetObjectReadSeekFunctional() { +func testSSECEncryptedGetObjectReadSeekFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -2675,7 +2804,7 @@ func testEncryptedGetObjectReadSeekFunctional() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -2683,7 +2812,7 @@ func testEncryptedGetObjectReadSeekFunctional() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -2704,9 +2833,9 @@ func testEncryptedGetObjectReadSeekFunctional() { } }() - // Generate 65MiB of data. - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") + // Generate 129MiB of data. + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") @@ -2844,8 +2973,194 @@ func testEncryptedGetObjectReadSeekFunctional() { successLogger(testName, function, args, startTime).Info() } +// Tests SSE-S3 get object ReaderSeeker interface methods. +func testSSES3EncryptedGetObjectReadSeekFunctional() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + defer func() { + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + }() + + // Generate 129MiB of data. + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + buf, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + // Save the data + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + ContentType: "binary/octet-stream", + ServerSideEncryption: encrypt.NewSSE(), + }) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + defer r.Close() + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat object failed", err) + return + } + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + return + } + + // This following function helps us to compare data from the reader after seek + // with the data from the original buffer + cmpData := func(r io.Reader, start, end int) { + if end-start == 0 { + return + } + buffer := bytes.NewBuffer([]byte{}) + if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "CopyN failed", err) + return + } + } + if !bytes.Equal(buf[start:end], buffer.Bytes()) { + logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) + return + } + } + + testCases := []struct { + offset int64 + whence int + pos int64 + err error + shouldCmp bool + start int + end int + }{ + // Start from offset 0, fetch data and compare + {0, 0, 0, nil, true, 0, 0}, + // Start from offset 2048, fetch data and compare + {2048, 0, 2048, nil, true, 2048, bufSize}, + // Start from offset larger than possible + {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, + // Move to offset 0 without comparing + {0, 0, 0, nil, false, 0, 0}, + // Move one step forward and compare + {1, 1, 1, nil, true, 1, bufSize}, + // Move larger than possible + {int64(bufSize), 1, 0, io.EOF, false, 0, 0}, + // Provide negative offset with CUR_SEEK + {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, + // Test with whence SEEK_END and with positive offset + {1024, 2, 0, io.EOF, false, 0, 0}, + // Test with whence SEEK_END and with negative offset + {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, + // Test with whence SEEK_END and with large negative offset + {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, + // Test with invalid whence + {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, + } + + for i, testCase := range testCases { + // Perform seek operation + n, err := r.Seek(testCase.offset, testCase.whence) + if err != nil && testCase.err == nil { + // We expected success. + logError(testName, function, args, startTime, "", + fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) + return + } + if err == nil && testCase.err != nil { + // We expected failure, but got success. + logError(testName, function, args, startTime, "", + fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) + return + } + if err != nil && testCase.err != nil { + if err.Error() != testCase.err.Error() { + // We expect a specific error + logError(testName, function, args, startTime, "", + fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) + return + } + } + // Check the returned seek pos + if n != testCase.pos { + logError(testName, function, args, startTime, "", + fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err) + return + } + // Compare only if shouldCmp is activated + if testCase.shouldCmp { + cmpData(r, testCase.start, testCase.end) + } + } + + successLogger(testName, function, args, startTime).Info() +} + // Tests SSE-C get object ReaderAt interface methods. -func testEncryptedGetObjectReadAtFunctional() { +func testSSECEncryptedGetObjectReadAtFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -2863,7 +3178,7 @@ func testEncryptedGetObjectReadAtFunctional() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -2871,7 +3186,7 @@ func testEncryptedGetObjectReadAtFunctional() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -2884,9 +3199,9 @@ func testEncryptedGetObjectReadAtFunctional() { return } - // Generate 65MiB of data. - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") + // Generate 129MiB of data. + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") @@ -3035,8 +3350,197 @@ func testEncryptedGetObjectReadAtFunctional() { successLogger(testName, function, args, startTime).Info() } -// TestEncryptionPutGet tests client side encryption -func testEncryptionPutGet() { +// Tests SSE-S3 get object ReaderAt interface methods. +func testSSES3EncryptedGetObjectReadAtFunctional() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate 129MiB of data. + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + buf, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + // Save the data + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + ContentType: "binary/octet-stream", + ServerSideEncryption: encrypt.NewSSE(), + }) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) + return + } + + // read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + defer r.Close() + + offset := int64(2048) + + // read directly + buf1 := make([]byte, 512) + buf2 := make([]byte, 512) + buf3 := make([]byte, 512) + buf4 := make([]byte, 512) + + // Test readAt before stat is called such that objectInfo doesn't change. + m, err := r.ReadAt(buf1, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf1) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) + return + } + if !bytes.Equal(buf1, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + offset += 512 + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + return + } + + m, err = r.ReadAt(buf2, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf2) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) + return + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + offset += 512 + m, err = r.ReadAt(buf3, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf3) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) + return + } + if !bytes.Equal(buf3, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + offset += 512 + m, err = r.ReadAt(buf4, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf4) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) + return + } + if !bytes.Equal(buf4, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + + buf5 := make([]byte, n) + // Read the whole object. + m, err = r.ReadAt(buf5, 0) + if err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + } + if m != len(buf5) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) + return + } + if !bytes.Equal(buf, buf5) { + logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) + return + } + + buf6 := make([]byte, n+1) + // Read the whole object and beyond. + _, err = r.ReadAt(buf6, 0) + if err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +// testSSECEncryptionPutGet tests encryption with customer provided encryption keys +func testSSECEncryptionPutGet() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -3057,7 +3561,7 @@ func testEncryptionPutGet() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -3065,7 +3569,7 @@ func testEncryptionPutGet() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -3147,8 +3651,8 @@ func testEncryptionPutGet() { successLogger(testName, function, args, startTime).Info() } -// TestEncryptionFPut tests client side encryption -func testEncryptionFPut() { +// TestEncryptionFPut tests encryption with customer specified encryption keys +func testSSECEncryptionFPut() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -3171,7 +3675,7 @@ func testEncryptionFPut() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -3179,7 +3683,7 @@ func testEncryptionFPut() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -3279,6 +3783,247 @@ func testEncryptionFPut() { successLogger(testName, function, args, startTime).Info() } +// testSSES3EncryptionPutGet tests SSE-S3 encryption +func testSSES3EncryptionPutGet() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutEncryptedObject(bucketName, objectName, reader, sse)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "sse": "", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + testCases := []struct { + buf []byte + }{ + {buf: bytes.Repeat([]byte("F"), 1)}, + {buf: bytes.Repeat([]byte("F"), 15)}, + {buf: bytes.Repeat([]byte("F"), 16)}, + {buf: bytes.Repeat([]byte("F"), 17)}, + {buf: bytes.Repeat([]byte("F"), 31)}, + {buf: bytes.Repeat([]byte("F"), 32)}, + {buf: bytes.Repeat([]byte("F"), 33)}, + {buf: bytes.Repeat([]byte("F"), 1024)}, + {buf: bytes.Repeat([]byte("F"), 1024*2)}, + {buf: bytes.Repeat([]byte("F"), 1024*1024)}, + } + + for i, testCase := range testCases { + // Generate a random object name + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Secured object + sse := encrypt.NewSSE() + args["sse"] = sse + + // Put encrypted data + _, err = c.PutObject(bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) + if err != nil { + logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) + return + } + + // Read the data back without any encryption headers + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) + return + } + defer r.Close() + + // Compare the sent object with the received one + recvBuffer := bytes.NewBuffer([]byte{}) + if _, err = io.Copy(recvBuffer, r); err != nil { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) + return + } + if recvBuffer.Len() != len(testCase.buf) { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) + return + } + if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) + return + } + + successLogger(testName, function, args, startTime).Info() + + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// TestSSES3EncryptionFPut tests server side encryption +func testSSES3EncryptionFPut() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "filePath": "", + "contentType": "", + "sse": "", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Object custom metadata + customContentType := "custom/contenttype" + args["metadata"] = customContentType + + testCases := []struct { + buf []byte + }{ + {buf: bytes.Repeat([]byte("F"), 0)}, + {buf: bytes.Repeat([]byte("F"), 1)}, + {buf: bytes.Repeat([]byte("F"), 15)}, + {buf: bytes.Repeat([]byte("F"), 16)}, + {buf: bytes.Repeat([]byte("F"), 17)}, + {buf: bytes.Repeat([]byte("F"), 31)}, + {buf: bytes.Repeat([]byte("F"), 32)}, + {buf: bytes.Repeat([]byte("F"), 33)}, + {buf: bytes.Repeat([]byte("F"), 1024)}, + {buf: bytes.Repeat([]byte("F"), 1024*2)}, + {buf: bytes.Repeat([]byte("F"), 1024*1024)}, + } + + for i, testCase := range testCases { + // Generate a random object name + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Secured object + sse := encrypt.NewSSE() + args["sse"] = sse + + // Generate a random file name. + fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + file, err := os.Create(fileName) + if err != nil { + logError(testName, function, args, startTime, "", "file create failed", err) + return + } + _, err = file.Write(testCase.buf) + if err != nil { + logError(testName, function, args, startTime, "", "file write failed", err) + return + } + file.Close() + // Put encrypted data + if _, err = c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { + logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) + return + } + defer r.Close() + + // Compare the sent object with the received one + recvBuffer := bytes.NewBuffer([]byte{}) + if _, err = io.Copy(recvBuffer, r); err != nil { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) + return + } + if recvBuffer.Len() != len(testCase.buf) { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) + return + } + if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) + return + } + + if err = os.Remove(fileName); err != nil { + logError(testName, function, args, startTime, "", "File remove failed", err) + return + } + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + func testBucketNotification() { // initialize logging params startTime := time.Now() @@ -3307,7 +4052,7 @@ func testBucketNotification() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -3315,7 +4060,7 @@ func testBucketNotification() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") bucketName := os.Getenv("NOTIFY_BUCKET") args["bucketName"] = bucketName @@ -3403,7 +4148,7 @@ func testFunctional() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, nil, startTime, "", "Minio client object creation failed", err) + logError(testName, function, nil, startTime, "", "MinIO client object creation failed", err) return } @@ -3411,7 +4156,7 @@ func testFunctional() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -4018,7 +4763,7 @@ func testGetObjectModified() { ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -4026,7 +4771,7 @@ func testGetObjectModified() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -4118,7 +4863,7 @@ func testPutObjectUploadSeekedObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -4126,7 +4871,7 @@ func testPutObjectUploadSeekedObject() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -4251,7 +4996,7 @@ func testMakeBucketErrorV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -4259,7 +5004,7 @@ func testMakeBucketErrorV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -4312,7 +5057,7 @@ func testGetObjectClosedTwiceV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -4320,7 +5065,7 @@ func testGetObjectClosedTwiceV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -4412,7 +5157,7 @@ func testFPutObjectV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -4420,7 +5165,7 @@ func testFPutObjectV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -4583,7 +5328,7 @@ func testMakeBucketRegionsV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -4591,7 +5336,7 @@ func testMakeBucketRegionsV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -4646,7 +5391,7 @@ func testGetObjectReadSeekFunctionalV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -4654,7 +5399,7 @@ func testGetObjectReadSeekFunctionalV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -4810,7 +5555,7 @@ func testGetObjectReadAtFunctionalV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -4818,7 +5563,7 @@ func testGetObjectReadAtFunctionalV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -4980,7 +5725,7 @@ func testCopyObjectV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -4988,7 +5733,7 @@ func testCopyObjectV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -5247,7 +5992,7 @@ func testComposeObjectErrorCasesV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -5342,7 +6087,7 @@ func testCompose10KSourcesV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -5364,7 +6109,7 @@ func testEncryptedEmptyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -5449,15 +6194,14 @@ func testEncryptedEmptyObject() { successLogger(testName, function, args, startTime).Info() } -func testEncryptedCopyObjectWrapper(c *minio.Client) { +func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, sseDst encrypt.ServerSide) { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} + var srcEncryption, dstEncryption encrypt.ServerSide - // Generate a new random bucket name. - bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err := c.MakeBucket(bucketName, "us-east-1") if err != nil { @@ -5465,9 +6209,6 @@ func testEncryptedCopyObjectWrapper(c *minio.Client) { return } - sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) - sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) - // 1. create an sse-c encrypted object to copy by uploading const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB @@ -5479,8 +6220,12 @@ func testEncryptedCopyObjectWrapper(c *minio.Client) { return } + if sseSrc != nil && sseSrc.Type() != encrypt.S3 { + srcEncryption = sseSrc + } + // 2. copy object and change encryption key - src := minio.NewSourceInfo(bucketName, "srcObject", sseSrc) + src := minio.NewSourceInfo(bucketName, "srcObject", srcEncryption) args["source"] = src dst, err := minio.NewDestinationInfo(bucketName, "dstObject", sseDst, nil) if err != nil { @@ -5495,9 +6240,12 @@ func testEncryptedCopyObjectWrapper(c *minio.Client) { return } + if sseDst != nil && sseDst.Type() != encrypt.S3 { + dstEncryption = sseDst + } // 3. get copied object and check if content is equal coreClient := minio.Core{c} - reader, _, err := coreClient.GetObject(bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: sseDst}) + reader, _, _, err := coreClient.GetObject(bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return @@ -5515,26 +6263,68 @@ func testEncryptedCopyObjectWrapper(c *minio.Client) { reader.Close() // Test key rotation for source object in-place. - newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"srcObject")) // replace key - dst, err = minio.NewDestinationInfo(bucketName, "srcObject", newSSE, nil) - if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) - return + var newSSE encrypt.ServerSide + if sseSrc != nil && sseSrc.Type() == encrypt.SSEC { + newSSE = encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"srcObject")) // replace key } - args["destination"] = dst + if sseSrc != nil && sseSrc.Type() == encrypt.S3 { + newSSE = encrypt.NewSSE() + } + if newSSE != nil { + dst, err = minio.NewDestinationInfo(bucketName, "srcObject", newSSE, nil) + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + args["destination"] = dst - err = c.CopyObject(dst, src) - if err != nil { - logError(testName, function, args, startTime, "", "CopyObject failed", err) - return + err = c.CopyObject(dst, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) + return + } + + // Get copied object and check if content is equal + reader, _, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + decBytes, err = ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + if !bytes.Equal(decBytes, buf) { + logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) + return + } + reader.Close() + // Test in-place decryption. + dst, err = minio.NewDestinationInfo(bucketName, "srcObject", nil, nil) + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + args["destination"] = dst + + src = minio.NewSourceInfo(bucketName, "srcObject", newSSE) + args["source"] = src + err = c.CopyObject(dst, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject Key rotation failed", err) + return + } } - // Get copied object and check if content is equal - reader, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE}) + // Get copied decrypted object and check if content is equal + reader, _, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } + defer reader.Close() decBytes, err = ioutil.ReadAll(reader) if err != nil { @@ -5545,53 +6335,213 @@ func testEncryptedCopyObjectWrapper(c *minio.Client) { logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) return } - reader.Close() - // Test in-place decryption. - dst, err = minio.NewDestinationInfo(bucketName, "srcObject", nil, nil) + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test encrypted copy object +func testUnencryptedToSSECCopyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) if err != nil { - logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } - args["destination"] = dst + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - src = minio.NewSourceInfo(bucketName, "srcObject", newSSE) - args["source"] = src - err = c.CopyObject(dst, src) + var sseSrc encrypt.ServerSide + sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) + // c.TraceOn(os.Stderr) + testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) +} + +// Test encrypted copy object +func testUnencryptedToSSES3CopyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) if err != nil { - logError(testName, function, args, startTime, "", "CopyObject failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - // Get copied decrypted object and check if content is equal - reader, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{}) + var sseSrc encrypt.ServerSide + sseDst := encrypt.NewSSE() + // c.TraceOn(os.Stderr) + testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) +} + +// Test encrypted copy object +func testUnencryptedToUnencryptedCopyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) if err != nil { - logError(testName, function, args, startTime, "", "GetObject failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } - defer reader.Close() + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - decBytes, err = ioutil.ReadAll(reader) + var sseSrc, sseDst encrypt.ServerSide + // c.TraceOn(os.Stderr) + testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) +} + +// Test encrypted copy object +func testEncryptedSSECToSSECCopyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) if err != nil { - logError(testName, function, args, startTime, "", "ReadAll failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } - if !bytes.Equal(decBytes, buf) { - logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + + sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) + sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) + // c.TraceOn(os.Stderr) + testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) +} + +// Test encrypted copy object +func testEncryptedSSECToSSES3CopyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - // Delete all objects and buckets - if err = cleanupBucket(bucketName, c); err != nil { - logError(testName, function, args, startTime, "", "Cleanup failed", err) + sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) + sseDst := encrypt.NewSSE() + // c.TraceOn(os.Stderr) + testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) +} + +// Test encrypted copy object +func testEncryptedSSECToUnencryptedCopyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") - successLogger(testName, function, args, startTime).Info() + sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) + var sseDst encrypt.ServerSide + // c.TraceOn(os.Stderr) + testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) +} + +// Test encrypted copy object +func testEncryptedSSES3ToSSECCopyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) + return + } + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + + sseSrc := encrypt.NewSSE() + sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) + // c.TraceOn(os.Stderr) + testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object -func testEncryptedCopyObject() { +func testEncryptedSSES3ToSSES3CopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -5606,12 +6556,44 @@ func testEncryptedCopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + sseSrc := encrypt.NewSSE() + sseDst := encrypt.NewSSE() // c.TraceOn(os.Stderr) - testEncryptedCopyObjectWrapper(c) + testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) +} + +// Test encrypted copy object +func testEncryptedSSES3ToUnencryptedCopyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) + return + } + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + + sseSrc := encrypt.NewSSE() + var sseDst encrypt.ServerSide + // c.TraceOn(os.Stderr) + testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object @@ -5630,12 +6612,16 @@ func testEncryptedCopyObjectV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) + sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) // c.TraceOn(os.Stderr) - testEncryptedCopyObjectWrapper(c) + testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } func testDecryptedCopyObject() { @@ -5653,7 +6639,7 @@ func testDecryptedCopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } @@ -5692,8 +6678,179 @@ func testDecryptedCopyObject() { successLogger(testName, function, args, startTime).Info() } +func testSSECMultipartEncryptedToSSECCopyObjectPart() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObjectPart(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + client, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) + return + } + + // Instantiate new core client object. + c := minio.Core{client} + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + } + defer cleanupBucket(bucketName, client) + // Make a buffer with 6MB of data + buf := bytes.Repeat([]byte("abcdef"), 1024*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + password := "correct horse battery staple" + srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) + + // Upload a 6MB object using multipart mechanism + uploadID, err := c.NewMultipartUpload(bucketName, objectName, minio.PutObjectOptions{ServerSideEncryption: srcencryption}) + if err != nil { + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) + } + + var completeParts []minio.CompletePart + + part, err := c.PutObjectPart(bucketName, objectName, uploadID, 1, bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024, "", "", srcencryption) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) + } + completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) + + part, err = c.PutObjectPart(bucketName, objectName, uploadID, 2, bytes.NewReader(buf[5*1024*1024:]), 1024*1024, "", "", srcencryption) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) + } + completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(bucketName, objectName, uploadID, completeParts) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } + + // Stat the object and check its length matches + objInfo, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: srcencryption}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + destBucketName := bucketName + destObjectName := objectName + "-dest" + dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) + + uploadID, err = c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + if err != nil { + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) + } + + // Content of the destination object will be two copies of + // `objectName` concatenated, followed by first byte of + // `objectName`. + metadata := make(map[string]string) + header := make(http.Header) + encrypt.SSECopy(srcencryption).Marshal(header) + dstencryption.Marshal(header) + for k, v := range header { + metadata[k] = v[0] + } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + + // First of three parts + fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Second of three parts + sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Last of three parts + lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } + + // Stat the object and check its length matches + objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if objInfo.Size != (6*1024*1024)*2+1 { + logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) + } + + // Now we read the data back + getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} + getOpts.SetRange(0, 6*1024*1024-1) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf := make([]byte, 6*1024*1024) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf, buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in first 6MB", err) + } + + getOpts.SetRange(6*1024*1024, 0) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf = make([]byte, 6*1024*1024+1) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf[:6*1024*1024], buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in second 6MB", err) + } + if getBuf[6*1024*1024] != buf[0] { + logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) + } + + successLogger(testName, function, args, startTime).Info() + + // Do not need to remove destBucketName its same as bucketName. +} + // Test Core CopyObjectPart implementation -func testCoreEncryptedCopyObjectPart() { +func testSSECEncryptedToSSECCopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -5708,7 +6865,7 @@ func testCoreEncryptedCopyObjectPart() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -5719,7 +6876,7 @@ func testCoreEncryptedCopyObjectPart() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") @@ -5768,6 +6925,460 @@ func testCoreEncryptedCopyObjectPart() { for k, v := range header { metadata[k] = v[0] } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + + // First of three parts + fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Second of three parts + sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Last of three parts + lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } + + // Stat the object and check its length matches + objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if objInfo.Size != (5*1024*1024)*2+1 { + logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) + } + + // Now we read the data back + getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} + getOpts.SetRange(0, 5*1024*1024-1) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf := make([]byte, 5*1024*1024) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf, buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) + } + + getOpts.SetRange(5*1024*1024, 0) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf = make([]byte, 5*1024*1024+1) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf[:5*1024*1024], buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) + } + if getBuf[5*1024*1024] != buf[0] { + logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) + } + + successLogger(testName, function, args, startTime).Info() + + // Do not need to remove destBucketName its same as bucketName. +} + +// Test Core CopyObjectPart implementation for SSEC encrypted to unencrypted copy +func testSSECEncryptedToUnencryptedCopyPart() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObjectPart(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + client, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) + return + } + + // Instantiate new core client object. + c := minio.Core{client} + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + } + defer cleanupBucket(bucketName, client) + // Make a buffer with 5MB of data + buf := bytes.Repeat([]byte("abcde"), 1024*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + password := "correct horse battery staple" + srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) + + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + "Content-Type": "binary/octet-stream", + }, srcencryption) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + } + + if objInfo.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + } + + destBucketName := bucketName + destObjectName := objectName + "-dest" + var dstencryption encrypt.ServerSide + + uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + if err != nil { + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) + } + + // Content of the destination object will be two copies of + // `objectName` concatenated, followed by first byte of + // `objectName`. + metadata := make(map[string]string) + header := make(http.Header) + encrypt.SSECopy(srcencryption).Marshal(header) + for k, v := range header { + metadata[k] = v[0] + } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + + // First of three parts + fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Second of three parts + sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Last of three parts + lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } + + // Stat the object and check its length matches + objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if objInfo.Size != (5*1024*1024)*2+1 { + logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) + } + + // Now we read the data back + getOpts := minio.GetObjectOptions{} + getOpts.SetRange(0, 5*1024*1024-1) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf := make([]byte, 5*1024*1024) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf, buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) + } + + getOpts.SetRange(5*1024*1024, 0) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf = make([]byte, 5*1024*1024+1) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf[:5*1024*1024], buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) + } + if getBuf[5*1024*1024] != buf[0] { + logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) + } + + successLogger(testName, function, args, startTime).Info() + + // Do not need to remove destBucketName its same as bucketName. +} + +// Test Core CopyObjectPart implementation for SSEC encrypted to SSE-S3 encrypted copy +func testSSECEncryptedToSSES3CopyObjectPart() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObjectPart(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + client, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) + return + } + + // Instantiate new core client object. + c := minio.Core{client} + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + } + defer cleanupBucket(bucketName, client) + // Make a buffer with 5MB of data + buf := bytes.Repeat([]byte("abcde"), 1024*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + password := "correct horse battery staple" + srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) + + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + "Content-Type": "binary/octet-stream", + }, srcencryption) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + } + + if objInfo.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + } + + destBucketName := bucketName + destObjectName := objectName + "-dest" + dstencryption := encrypt.NewSSE() + + uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + if err != nil { + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) + } + + // Content of the destination object will be two copies of + // `objectName` concatenated, followed by first byte of + // `objectName`. + metadata := make(map[string]string) + header := make(http.Header) + encrypt.SSECopy(srcencryption).Marshal(header) + dstencryption.Marshal(header) + + for k, v := range header { + metadata[k] = v[0] + } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + + // First of three parts + fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Second of three parts + sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Last of three parts + lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } + + // Stat the object and check its length matches + objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if objInfo.Size != (5*1024*1024)*2+1 { + logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) + } + + // Now we read the data back + getOpts := minio.GetObjectOptions{} + getOpts.SetRange(0, 5*1024*1024-1) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf := make([]byte, 5*1024*1024) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf, buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) + } + + getOpts.SetRange(5*1024*1024, 0) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf = make([]byte, 5*1024*1024+1) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf[:5*1024*1024], buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) + } + if getBuf[5*1024*1024] != buf[0] { + logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) + } + + successLogger(testName, function, args, startTime).Info() + + // Do not need to remove destBucketName its same as bucketName. +} + +// Test Core CopyObjectPart implementation for unencrypted to SSEC encryption copy part +func testUnencryptedToSSECCopyObjectPart() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObjectPart(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + client, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) + return + } + + // Instantiate new core client object. + c := minio.Core{client} + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + } + defer cleanupBucket(bucketName, client) + // Make a buffer with 5MB of data + buf := bytes.Repeat([]byte("abcde"), 1024*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + password := "correct horse battery staple" + + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + "Content-Type": "binary/octet-stream", + }, nil) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + } + + if objInfo.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + } + + destBucketName := bucketName + destObjectName := objectName + "-dest" + dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) + + uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + if err != nil { + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) + } + + // Content of the destination object will be two copies of + // `objectName` concatenated, followed by first byte of + // `objectName`. + metadata := make(map[string]string) + header := make(http.Header) + dstencryption.Marshal(header) + for k, v := range header { + metadata[k] = v[0] + } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + // First of three parts fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { @@ -5805,7 +7416,7 @@ func testCoreEncryptedCopyObjectPart() { // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 5*1024*1024-1) - r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -5819,7 +7430,748 @@ func testCoreEncryptedCopyObjectPart() { } getOpts.SetRange(5*1024*1024, 0) - r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf = make([]byte, 5*1024*1024+1) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf[:5*1024*1024], buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) + } + if getBuf[5*1024*1024] != buf[0] { + logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) + } + + successLogger(testName, function, args, startTime).Info() + + // Do not need to remove destBucketName its same as bucketName. +} + +// Test Core CopyObjectPart implementation for unencrypted to unencrypted copy +func testUnencryptedToUnencryptedCopyPart() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObjectPart(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + client, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) + return + } + + // Instantiate new core client object. + c := minio.Core{client} + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + } + defer cleanupBucket(bucketName, client) + // Make a buffer with 5MB of data + buf := bytes.Repeat([]byte("abcde"), 1024*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + "Content-Type": "binary/octet-stream", + }, nil) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + } + + if objInfo.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + } + + destBucketName := bucketName + destObjectName := objectName + "-dest" + + uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) + } + + // Content of the destination object will be two copies of + // `objectName` concatenated, followed by first byte of + // `objectName`. + metadata := make(map[string]string) + header := make(http.Header) + for k, v := range header { + metadata[k] = v[0] + } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + + // First of three parts + fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Second of three parts + sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Last of three parts + lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } + + // Stat the object and check its length matches + objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if objInfo.Size != (5*1024*1024)*2+1 { + logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) + } + + // Now we read the data back + getOpts := minio.GetObjectOptions{} + getOpts.SetRange(0, 5*1024*1024-1) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf := make([]byte, 5*1024*1024) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf, buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) + } + + getOpts.SetRange(5*1024*1024, 0) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf = make([]byte, 5*1024*1024+1) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf[:5*1024*1024], buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) + } + if getBuf[5*1024*1024] != buf[0] { + logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) + } + + successLogger(testName, function, args, startTime).Info() + + // Do not need to remove destBucketName its same as bucketName. +} + +// Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy +func testUnencryptedToSSES3CopyObjectPart() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObjectPart(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + client, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) + return + } + + // Instantiate new core client object. + c := minio.Core{client} + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + } + defer cleanupBucket(bucketName, client) + // Make a buffer with 5MB of data + buf := bytes.Repeat([]byte("abcde"), 1024*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + "Content-Type": "binary/octet-stream", + }, nil) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + } + + if objInfo.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + } + + destBucketName := bucketName + destObjectName := objectName + "-dest" + dstencryption := encrypt.NewSSE() + + uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + if err != nil { + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) + } + + // Content of the destination object will be two copies of + // `objectName` concatenated, followed by first byte of + // `objectName`. + metadata := make(map[string]string) + header := make(http.Header) + dstencryption.Marshal(header) + + for k, v := range header { + metadata[k] = v[0] + } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + + // First of three parts + fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Second of three parts + sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Last of three parts + lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } + + // Stat the object and check its length matches + objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if objInfo.Size != (5*1024*1024)*2+1 { + logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) + } + + // Now we read the data back + getOpts := minio.GetObjectOptions{} + getOpts.SetRange(0, 5*1024*1024-1) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf := make([]byte, 5*1024*1024) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf, buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) + } + + getOpts.SetRange(5*1024*1024, 0) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf = make([]byte, 5*1024*1024+1) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf[:5*1024*1024], buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) + } + if getBuf[5*1024*1024] != buf[0] { + logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) + } + + successLogger(testName, function, args, startTime).Info() + + // Do not need to remove destBucketName its same as bucketName. +} + +// Test Core CopyObjectPart implementation for SSE-S3 to SSEC encryption copy part +func testSSES3EncryptedToSSECCopyObjectPart() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObjectPart(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + client, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) + return + } + + // Instantiate new core client object. + c := minio.Core{client} + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + } + defer cleanupBucket(bucketName, client) + // Make a buffer with 5MB of data + buf := bytes.Repeat([]byte("abcde"), 1024*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + password := "correct horse battery staple" + srcEncryption := encrypt.NewSSE() + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + "Content-Type": "binary/octet-stream", + }, srcEncryption) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + } + + if objInfo.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + } + + destBucketName := bucketName + destObjectName := objectName + "-dest" + dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) + + uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + if err != nil { + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) + } + + // Content of the destination object will be two copies of + // `objectName` concatenated, followed by first byte of + // `objectName`. + metadata := make(map[string]string) + header := make(http.Header) + dstencryption.Marshal(header) + for k, v := range header { + metadata[k] = v[0] + } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + + // First of three parts + fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Second of three parts + sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Last of three parts + lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } + + // Stat the object and check its length matches + objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if objInfo.Size != (5*1024*1024)*2+1 { + logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) + } + + // Now we read the data back + getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} + getOpts.SetRange(0, 5*1024*1024-1) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf := make([]byte, 5*1024*1024) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf, buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) + } + + getOpts.SetRange(5*1024*1024, 0) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf = make([]byte, 5*1024*1024+1) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf[:5*1024*1024], buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) + } + if getBuf[5*1024*1024] != buf[0] { + logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) + } + + successLogger(testName, function, args, startTime).Info() + + // Do not need to remove destBucketName its same as bucketName. +} + +// Test Core CopyObjectPart implementation for unencrypted to unencrypted copy +func testSSES3EncryptedToUnencryptedCopyPart() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObjectPart(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + client, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) + return + } + + // Instantiate new core client object. + c := minio.Core{client} + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + } + defer cleanupBucket(bucketName, client) + // Make a buffer with 5MB of data + buf := bytes.Repeat([]byte("abcde"), 1024*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + srcEncryption := encrypt.NewSSE() + + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + "Content-Type": "binary/octet-stream", + }, srcEncryption) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + } + + if objInfo.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + } + + destBucketName := bucketName + destObjectName := objectName + "-dest" + + uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) + } + + // Content of the destination object will be two copies of + // `objectName` concatenated, followed by first byte of + // `objectName`. + metadata := make(map[string]string) + header := make(http.Header) + for k, v := range header { + metadata[k] = v[0] + } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + + // First of three parts + fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Second of three parts + sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Last of three parts + lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } + + // Stat the object and check its length matches + objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if objInfo.Size != (5*1024*1024)*2+1 { + logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) + } + + // Now we read the data back + getOpts := minio.GetObjectOptions{} + getOpts.SetRange(0, 5*1024*1024-1) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf := make([]byte, 5*1024*1024) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf, buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) + } + + getOpts.SetRange(5*1024*1024, 0) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf = make([]byte, 5*1024*1024+1) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf[:5*1024*1024], buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) + } + if getBuf[5*1024*1024] != buf[0] { + logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) + } + + successLogger(testName, function, args, startTime).Info() + + // Do not need to remove destBucketName its same as bucketName. +} + +// Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy +func testSSES3EncryptedToSSES3CopyObjectPart() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObjectPart(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + client, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) + return + } + + // Instantiate new core client object. + c := minio.Core{client} + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + } + defer cleanupBucket(bucketName, client) + // Make a buffer with 5MB of data + buf := bytes.Repeat([]byte("abcde"), 1024*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + srcEncryption := encrypt.NewSSE() + + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + "Content-Type": "binary/octet-stream", + }, srcEncryption) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + } + + if objInfo.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + } + + destBucketName := bucketName + destObjectName := objectName + "-dest" + dstencryption := encrypt.NewSSE() + + uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + if err != nil { + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) + } + + // Content of the destination object will be two copies of + // `objectName` concatenated, followed by first byte of + // `objectName`. + metadata := make(map[string]string) + header := make(http.Header) + dstencryption.Marshal(header) + + for k, v := range header { + metadata[k] = v[0] + } + + metadata["x-amz-copy-source-if-match"] = objInfo.ETag + + // First of three parts + fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Second of three parts + sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Last of three parts + lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } + + // Stat the object and check its length matches + objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if objInfo.Size != (5*1024*1024)*2+1 { + logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) + } + + // Now we read the data back + getOpts := minio.GetObjectOptions{} + getOpts.SetRange(0, 5*1024*1024-1) + r, _, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf := make([]byte, 5*1024*1024) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf, buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) + } + + getOpts.SetRange(5*1024*1024, 0) + r, _, _, err = c.GetObject(destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) } @@ -5854,7 +8206,7 @@ func testUserMetadataCopying() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -6041,7 +8393,7 @@ func testUserMetadataCopyingV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -6064,7 +8416,7 @@ func testStorageClassMetadataPutObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -6154,7 +8506,7 @@ func testStorageClassInvalidMetadataPutObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -6199,7 +8551,7 @@ func testStorageClassMetadataCopyObject() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } @@ -6214,6 +8566,8 @@ func testStorageClassMetadataCopyObject() { fetchMeta := func(object string) (h http.Header) { objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) + args["bucket"] = bucketName + args["object"] = object if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return @@ -6248,7 +8602,9 @@ func testStorageClassMetadataCopyObject() { // Make server side copy of object uploaded in previous step src := minio.NewSourceInfo(bucketName, "srcObjectRRSClass", nil) dst, err := minio.NewDestinationInfo(bucketName, "srcObjectRRSClassCopy", nil, nil) - c.CopyObject(dst, src) + if err = c.CopyObject(dst, src); err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed on RRS", err) + } // Get the returned metadata returnedMeta := fetchMeta("srcObjectRRSClassCopy") @@ -6273,8 +8629,9 @@ func testStorageClassMetadataCopyObject() { // Make server side copy of object uploaded in previous step src = minio.NewSourceInfo(bucketName, "srcObjectSSClass", nil) dst, err = minio.NewDestinationInfo(bucketName, "srcObjectSSClassCopy", nil, nil) - c.CopyObject(dst, src) - + if err = c.CopyObject(dst, src); err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed on SS", err) + } // Fetch the meta data of copied object if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClassCopy")) { logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) @@ -6312,7 +8669,7 @@ func testPutObjectNoLengthV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -6320,7 +8677,7 @@ func testPutObjectNoLengthV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -6336,8 +8693,8 @@ func testPutObjectNoLengthV2() { objectName := bucketName + "unique" args["objectName"] = objectName - bufSize := dataFileMap["datafile-65-MB"] - var reader = getDataReader("datafile-65-MB") + bufSize := dataFileMap["datafile-129-MB"] + var reader = getDataReader("datafile-129-MB") defer reader.Close() args["size"] = bufSize @@ -6386,7 +8743,7 @@ func testPutObjectsUnknownV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -6394,7 +8751,7 @@ func testPutObjectsUnknownV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -6470,7 +8827,7 @@ func testPutObject0ByteV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -6478,7 +8835,7 @@ func testPutObject0ByteV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -6532,7 +8889,7 @@ func testComposeObjectErrorCases() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -6555,7 +8912,7 @@ func testCompose10KSources() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } @@ -6581,7 +8938,7 @@ func testFunctionalV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -6589,7 +8946,7 @@ func testFunctionalV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -6994,7 +9351,7 @@ func testGetObjectWithContext() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } @@ -7002,7 +9359,7 @@ func testGetObjectWithContext() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -7102,7 +9459,7 @@ func testFGetObjectWithContext() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } @@ -7110,7 +9467,7 @@ func testFGetObjectWithContext() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -7198,7 +9555,7 @@ func testGetObjectACL() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } @@ -7206,7 +9563,7 @@ func testGetObjectACL() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -7353,7 +9710,7 @@ func testPutObjectWithContextV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -7361,7 +9718,7 @@ func testPutObjectWithContextV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -7435,7 +9792,7 @@ func testGetObjectWithContextV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -7443,7 +9800,7 @@ func testGetObjectWithContextV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -7541,7 +9898,7 @@ func testFGetObjectWithContextV2() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } @@ -7549,7 +9906,7 @@ func testFGetObjectWithContextV2() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -7612,7 +9969,7 @@ func testFGetObjectWithContextV2() { } -// Test list object v1 and V2 storage class fields +// Test list object v1 and V2 func testListObjects() { // initialize logging params startTime := time.Now() @@ -7634,7 +9991,7 @@ func testListObjects() { mustParseBool(os.Getenv(enableHTTPS)), ) if err != nil { - logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } @@ -7642,7 +9999,7 @@ func testListObjects() { // c.TraceOn(os.Stderr) // Set user agent. - c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") @@ -7655,67 +10012,64 @@ func testListObjects() { return } - bufSize := dataFileMap["datafile-33-kB"] - var reader = getDataReader("datafile-33-kB") - defer reader.Close() - - // Save the data - objectName1 := randString(60, rand.NewSource(time.Now().UnixNano()), "") + testObjects := []struct { + name string + storageClass string + }{ - _, err = c.PutObject(bucketName, objectName1, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: "STANDARD"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject1 call failed", err) - return + // \x17 is a forbidden character in a xml document + {"foo\x17bar", "STANDARD"}, + // Special characters + {"foo bar", "STANDARD"}, + {"foo-%", "STANDARD"}, + {"random-object-1", "STANDARD"}, + {"random-object-2", "REDUCED_REDUNDANCY"}, + } + + for i, object := range testObjects { + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + _, err = c.PutObject(bucketName, object.name, reader, int64(bufSize), + minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: object.storageClass}) + if err != nil { + logError(testName, function, args, startTime, "", fmt.Sprintf("PutObject %d call failed", i+1), err) + return + } } - bufSize1 := dataFileMap["datafile-33-kB"] - var reader1 = getDataReader("datafile-33-kB") - defer reader1.Close() - objectName2 := randString(60, rand.NewSource(time.Now().UnixNano()), "") - - _, err = c.PutObject(bucketName, objectName2, reader1, int64(bufSize1), minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: "REDUCED_REDUNDANCY"}) - if err != nil { - logError(testName, function, args, startTime, "", "PutObject2 call failed", err) - return - } + testList := func(listFn func(string, string, bool, <-chan struct{}) <-chan minio.ObjectInfo, bucket string) { + // Create a done channel to control 'ListObjects' go routine. + doneCh := make(chan struct{}) + // Exit cleanly upon return. + defer close(doneCh) - // Create a done channel to control 'ListObjects' go routine. - doneCh := make(chan struct{}) - // Exit cleanly upon return. - defer close(doneCh) + var objCursor int - // check for storage-class from ListObjects result - for objInfo := range c.ListObjects(bucketName, "", true, doneCh) { - if objInfo.Err != nil { - logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err) - return - } - if objInfo.Key == objectName1 && objInfo.StorageClass != "STANDARD" { - // Ignored as Gateways (Azure/GCS etc) wont return storage class - ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info() - } - if objInfo.Key == objectName2 && objInfo.StorageClass != "REDUCED_REDUNDANCY" { - // Ignored as Gateways (Azure/GCS etc) wont return storage class - ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info() + // check for object name and storage-class from listing object result + for objInfo := range listFn(bucket, "", true, doneCh) { + if objInfo.Err != nil { + logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err) + return + } + if objInfo.Key != testObjects[objCursor].name { + logError(testName, function, args, startTime, "", "ListObjects does not return expected object name", err) + } + if objInfo.StorageClass != testObjects[objCursor].storageClass { + // Ignored as Gateways (Azure/GCS etc) wont return storage class + ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info() + } + objCursor++ } - } - // check for storage-class from ListObjectsV2 result - for objInfo := range c.ListObjectsV2(bucketName, "", true, doneCh) { - if objInfo.Err != nil { - logError(testName, function, args, startTime, "", "ListObjectsV2 failed unexpectedly", err) - return - } - if objInfo.Key == objectName1 && objInfo.StorageClass != "STANDARD" { - // Ignored as Gateways (Azure/GCS etc) wont return storage class - ignoredLog(testName, function, args, startTime, "ListObjectsV2 doesn't return expected storage class").Info() - } - if objInfo.Key == objectName2 && objInfo.StorageClass != "REDUCED_REDUNDANCY" { - // Ignored as Gateways (Azure/GCS etc) wont return storage class - ignoredLog(testName, function, args, startTime, "ListObjectsV2 doesn't return expected storage class").Info() + if objCursor != len(testObjects) { + logError(testName, function, args, startTime, "", "ListObjects returned unexpected number of items", errors.New("")) } } + testList(c.ListObjects, bucketName) + testList(c.ListObjectsV2, bucketName) + // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "Cleanup failed", err) @@ -7723,7 +10077,6 @@ func testListObjects() { } successLogger(testName, function, args, startTime).Info() - } // Convert string to bool and always return false if any error @@ -7746,6 +10099,7 @@ func main() { log.SetLevel(log.InfoLevel) tls := mustParseBool(os.Getenv(enableHTTPS)) + kmsEnabled := mustParseBool(os.Getenv(enableKMS)) // execute tests if isFullMode() { testMakeBucketErrorV2() @@ -7778,6 +10132,7 @@ func main() { testFPutObject() testGetObjectReadSeekFunctional() testGetObjectReadAtFunctional() + testGetObjectReadAtWhenEOFWasReached() testPresignedPostPolicy() testCopyObject() testComposeObjectErrorCases() @@ -7790,7 +10145,9 @@ func main() { testGetObjectWithContext() testFPutObjectWithContext() testFGetObjectWithContext() + testGetObjectACL() + testPutObjectWithContext() testStorageClassMetadataPutObject() testStorageClassInvalidMetadataPutObject() @@ -7800,15 +10157,38 @@ func main() { // SSE-C tests will only work over TLS connection. if tls { - testEncryptionPutGet() - testEncryptionFPut() - testEncryptedGetObjectReadAtFunctional() - testEncryptedGetObjectReadSeekFunctional() + testSSECEncryptionPutGet() + testSSECEncryptionFPut() + testSSECEncryptedGetObjectReadAtFunctional() + testSSECEncryptedGetObjectReadSeekFunctional() testEncryptedCopyObjectV2() - testEncryptedCopyObject() + testEncryptedSSECToSSECCopyObject() + testEncryptedSSECToUnencryptedCopyObject() + testUnencryptedToSSECCopyObject() + testUnencryptedToUnencryptedCopyObject() testEncryptedEmptyObject() testDecryptedCopyObject() - testCoreEncryptedCopyObjectPart() + testSSECEncryptedToSSECCopyObjectPart() + testSSECMultipartEncryptedToSSECCopyObjectPart() + testSSECEncryptedToUnencryptedCopyPart() + testUnencryptedToSSECCopyObjectPart() + testUnencryptedToUnencryptedCopyPart() + if kmsEnabled { + testSSES3EncryptionPutGet() + testSSES3EncryptionFPut() + testSSES3EncryptedGetObjectReadAtFunctional() + testSSES3EncryptedGetObjectReadSeekFunctional() + testEncryptedSSECToSSES3CopyObject() + testEncryptedSSES3ToSSECCopyObject() + testEncryptedSSES3ToSSES3CopyObject() + testEncryptedSSES3ToUnencryptedCopyObject() + testUnencryptedToSSES3CopyObject() + testSSECEncryptedToSSES3CopyObjectPart() + testUnencryptedToSSES3CopyObjectPart() + testSSES3EncryptedToSSECCopyObjectPart() + testSSES3EncryptedToUnencryptedCopyPart() + testSSES3EncryptedToSSES3CopyObjectPart() + } } } else { testFunctional() diff --git a/get-options_test.go b/get-options_test.go index c5344a0..a447f09 100644 --- a/get-options_test.go +++ b/get-options_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -0,0 +1,14 @@ +module github.com/minio/minio-go/v6 + +go 1.12 + +require ( + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/minio/sha256-simd v0.1.1 + github.com/mitchellh/go-homedir v1.1.0 + github.com/sirupsen/logrus v1.4.2 // indirect + github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect + golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f + golang.org/x/net v0.0.0-20190522155817-f3200d17e092 + gopkg.in/ini.v1 v1.42.0 +) @@ -0,0 +1,41 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/hook-reader.go b/hook-reader.go index 8f32291..f251c1e 100644 --- a/hook-reader.go +++ b/hook-reader.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,10 @@ package minio -import "io" +import ( + "fmt" + "io" +) // hookReader hooks additional reader in the source stream. It is // useful for making progress bars. Second reader is appropriately @@ -34,12 +37,23 @@ func (hr *hookReader) Seek(offset int64, whence int) (n int64, err error) { // Verify for source has embedded Seeker, use it. sourceSeeker, ok := hr.source.(io.Seeker) if ok { - return sourceSeeker.Seek(offset, whence) + n, err = sourceSeeker.Seek(offset, whence) + if err != nil { + return 0, err + } } + // Verify if hook has embedded Seeker, use it. hookSeeker, ok := hr.hook.(io.Seeker) if ok { - return hookSeeker.Seek(offset, whence) + var m int64 + m, err = hookSeeker.Seek(offset, whence) + if err != nil { + return 0, err + } + if n != m { + return 0, fmt.Errorf("hook seeker seeked %d bytes, expected source %d bytes", m, n) + } } return n, nil } diff --git a/pkg/credentials/chain.go b/pkg/credentials/chain.go index e29826f..6dc8e9d 100644 --- a/pkg/credentials/chain.go +++ b/pkg/credentials/chain.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/chain_test.go b/pkg/credentials/chain_test.go index d26e376..2e18c29 100644 --- a/pkg/credentials/chain_test.go +++ b/pkg/credentials/chain_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/config.json.sample b/pkg/credentials/config.json.sample index 130746f..d793c9e 100644 --- a/pkg/credentials/config.json.sample +++ b/pkg/credentials/config.json.sample @@ -2,7 +2,7 @@ "version": "8", "hosts": { "play": { - "url": "https://play.minio.io:9000", + "url": "https://play.min.io", "accessKey": "Q3AM3UQ867SPQQA43P2F", "secretKey": "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "api": "S3v2" diff --git a/pkg/credentials/credentials.go b/pkg/credentials/credentials.go index 4bfdad4..1a48751 100644 --- a/pkg/credentials/credentials.go +++ b/pkg/credentials/credentials.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/credentials_test.go b/pkg/credentials/credentials_test.go index 92c77c4..57ea679 100644 --- a/pkg/credentials/credentials_test.go +++ b/pkg/credentials/credentials_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/doc.go b/pkg/credentials/doc.go index c48784b..0c94477 100644 --- a/pkg/credentials/doc.go +++ b/pkg/credentials/doc.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/env_aws.go b/pkg/credentials/env_aws.go index f9b2cc3..b6e60d0 100644 --- a/pkg/credentials/env_aws.go +++ b/pkg/credentials/env_aws.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/env_minio.go b/pkg/credentials/env_minio.go index d72e771..5f1ae0d 100644 --- a/pkg/credentials/env_minio.go +++ b/pkg/credentials/env_minio.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/env_test.go b/pkg/credentials/env_test.go index 09cd77f..2f982ae 100644 --- a/pkg/credentials/env_test.go +++ b/pkg/credentials/env_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/file_aws_credentials.go b/pkg/credentials/file_aws_credentials.go index 5ad6830..ff07bc5 100644 --- a/pkg/credentials/file_aws_credentials.go +++ b/pkg/credentials/file_aws_credentials.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,8 +21,8 @@ import ( "os" "path/filepath" - "github.com/go-ini/ini" homedir "github.com/mitchellh/go-homedir" + ini "gopkg.in/ini.v1" ) // A FileAWSCredentials retrieves credentials from the current user's home diff --git a/pkg/credentials/file_minio_client.go b/pkg/credentials/file_minio_client.go index 6a6827e..117ceb6 100644 --- a/pkg/credentials/file_minio_client.go +++ b/pkg/credentials/file_minio_client.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,7 +40,7 @@ type FileMinioClient struct { // Windows: "%USERALIAS%\mc\config.json" filename string - // Minio Alias to extract credentials from the shared credentials file. If empty + // MinIO Alias to extract credentials from the shared credentials file. If empty // will default to environment variable "MINIO_ALIAS" or "default" if // environment variable is also not set. alias string diff --git a/pkg/credentials/file_test.go b/pkg/credentials/file_test.go index c85c104..598c3f5 100644 --- a/pkg/credentials/file_test.go +++ b/pkg/credentials/file_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/iam_aws.go b/pkg/credentials/iam_aws.go index 6845c9a..5732f2e 100644 --- a/pkg/credentials/iam_aws.go +++ b/pkg/credentials/iam_aws.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -53,7 +53,7 @@ type IAM struct { const ( defaultIAMRoleEndpoint = "http://169.254.169.254" defaultECSRoleEndpoint = "http://169.254.170.2" - defaultIAMSecurityCredsPath = "/latest/meta-data/iam/security-credentials" + defaultIAMSecurityCredsPath = "/latest/meta-data/iam/security-credentials/" ) // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html @@ -67,9 +67,7 @@ func getEndpoint(endpoint string) (string, bool) { return defaultIAMRoleEndpoint, false } -// NewIAM returns a pointer to a new Credentials object wrapping -// the IAM. Takes a ConfigProvider to create a EC2Metadata client. -// The ConfigProvider is satisfied by the session.Session type. +// NewIAM returns a pointer to a new Credentials object wrapping the IAM. func NewIAM(endpoint string) *Credentials { p := &IAM{ Client: &http.Client{ diff --git a/pkg/credentials/iam_aws_test.go b/pkg/credentials/iam_aws_test.go index 4dbbb0a..90f9806 100644 --- a/pkg/credentials/iam_aws_test.go +++ b/pkg/credentials/iam_aws_test.go @@ -1,6 +1,8 @@ +// +build !windows + /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -65,11 +67,11 @@ func initTestServerNoRoles() *httptest.Server { func initTestServer(expireOn string, failAssume bool) *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/latest/meta-data/iam/security-credentials" { + if r.URL.Path == "/latest/meta-data/iam/security-credentials/" { fmt.Fprintln(w, "RoleName") } else if r.URL.Path == "/latest/meta-data/iam/security-credentials/RoleName" { if failAssume { - fmt.Fprintf(w, credsFailRespTmpl) + fmt.Fprint(w, credsFailRespTmpl) } else { fmt.Fprintf(w, credsRespTmpl, expireOn) } diff --git a/pkg/credentials/signature-type.go b/pkg/credentials/signature-type.go index 1b768e8..b794333 100644 --- a/pkg/credentials/signature-type.go +++ b/pkg/credentials/signature-type.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/static.go b/pkg/credentials/static.go index 8b0ba71..7dde00b 100644 --- a/pkg/credentials/static.go +++ b/pkg/credentials/static.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/static_test.go b/pkg/credentials/static_test.go index f1d2d85..65bec05 100644 --- a/pkg/credentials/static_test.go +++ b/pkg/credentials/static_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/credentials/sts_client_grants.go b/pkg/credentials/sts_client_grants.go new file mode 100644 index 0000000..03134c3 --- /dev/null +++ b/pkg/credentials/sts_client_grants.go @@ -0,0 +1,162 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "encoding/xml" + "errors" + "fmt" + "net/http" + "net/url" + "time" +) + +// AssumedRoleUser - The identifiers for the temporary security credentials that +// the operation returns. Please also see https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/AssumedRoleUser +type AssumedRoleUser struct { + Arn string + AssumedRoleID string `xml:"AssumeRoleId"` +} + +// AssumeRoleWithClientGrantsResponse contains the result of successful AssumeRoleWithClientGrants request. +type AssumeRoleWithClientGrantsResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithClientGrantsResponse" json:"-"` + Result ClientGrantsResult `xml:"AssumeRoleWithClientGrantsResult"` + ResponseMetadata struct { + RequestID string `xml:"RequestId,omitempty"` + } `xml:"ResponseMetadata,omitempty"` +} + +// ClientGrantsResult - Contains the response to a successful AssumeRoleWithClientGrants +// request, including temporary credentials that can be used to make MinIO API requests. +type ClientGrantsResult struct { + AssumedRoleUser AssumedRoleUser `xml:",omitempty"` + Audience string `xml:",omitempty"` + Credentials struct { + AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"` + SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"` + Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"` + SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"` + } `xml:",omitempty"` + PackedPolicySize int `xml:",omitempty"` + Provider string `xml:",omitempty"` + SubjectFromClientGrantsToken string `xml:",omitempty"` +} + +// ClientGrantsToken - client grants token with expiry. +type ClientGrantsToken struct { + Token string + Expiry int +} + +// A STSClientGrants retrieves credentials from MinIO service, and keeps track if +// those credentials are expired. +type STSClientGrants struct { + Expiry + + // Required http Client to use when connecting to MinIO STS service. + Client *http.Client + + // MinIO endpoint to fetch STS credentials. + stsEndpoint string + + // getClientGrantsTokenExpiry function to retrieve tokens + // from IDP This function should return two values one is + // accessToken which is a self contained access token (JWT) + // and second return value is the expiry associated with + // this token. This is a customer provided function and + // is mandatory. + getClientGrantsTokenExpiry func() (*ClientGrantsToken, error) +} + +// NewSTSClientGrants returns a pointer to a new +// Credentials object wrapping the STSClientGrants. +func NewSTSClientGrants(stsEndpoint string, getClientGrantsTokenExpiry func() (*ClientGrantsToken, error)) (*Credentials, error) { + if stsEndpoint == "" { + return nil, errors.New("STS endpoint cannot be empty") + } + if getClientGrantsTokenExpiry == nil { + return nil, errors.New("Client grants access token and expiry retrieval function should be defined") + } + return New(&STSClientGrants{ + Client: &http.Client{ + Transport: http.DefaultTransport, + }, + stsEndpoint: stsEndpoint, + getClientGrantsTokenExpiry: getClientGrantsTokenExpiry, + }), nil +} + +func getClientGrantsCredentials(clnt *http.Client, endpoint string, + getClientGrantsTokenExpiry func() (*ClientGrantsToken, error)) (AssumeRoleWithClientGrantsResponse, error) { + + accessToken, err := getClientGrantsTokenExpiry() + if err != nil { + return AssumeRoleWithClientGrantsResponse{}, err + } + + v := url.Values{} + v.Set("Action", "AssumeRoleWithClientGrants") + v.Set("Token", accessToken.Token) + v.Set("DurationSeconds", fmt.Sprintf("%d", accessToken.Expiry)) + v.Set("Version", "2011-06-15") + + u, err := url.Parse(endpoint) + if err != nil { + return AssumeRoleWithClientGrantsResponse{}, err + } + u.RawQuery = v.Encode() + + req, err := http.NewRequest("POST", u.String(), nil) + if err != nil { + return AssumeRoleWithClientGrantsResponse{}, err + } + resp, err := clnt.Do(req) + if err != nil { + return AssumeRoleWithClientGrantsResponse{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return AssumeRoleWithClientGrantsResponse{}, errors.New(resp.Status) + } + + a := AssumeRoleWithClientGrantsResponse{} + if err = xml.NewDecoder(resp.Body).Decode(&a); err != nil { + return AssumeRoleWithClientGrantsResponse{}, err + } + return a, nil +} + +// Retrieve retrieves credentials from the MinIO service. +// Error will be returned if the request fails. +func (m *STSClientGrants) Retrieve() (Value, error) { + a, err := getClientGrantsCredentials(m.Client, m.stsEndpoint, m.getClientGrantsTokenExpiry) + if err != nil { + return Value{}, err + } + + // Expiry window is set to 10secs. + m.SetExpiration(a.Result.Credentials.Expiration, DefaultExpiryWindow) + + return Value{ + AccessKeyID: a.Result.Credentials.AccessKey, + SecretAccessKey: a.Result.Credentials.SecretKey, + SessionToken: a.Result.Credentials.SessionToken, + SignerType: SignatureV4, + }, nil +} diff --git a/pkg/credentials/sts_ldap_identity.go b/pkg/credentials/sts_ldap_identity.go new file mode 100644 index 0000000..b72ac06 --- /dev/null +++ b/pkg/credentials/sts_ldap_identity.go @@ -0,0 +1,119 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "encoding/xml" + "errors" + "net/http" + "net/url" + "time" +) + +// AssumeRoleWithLDAPResponse contains the result of successful +// AssumeRoleWithLDAPIdentity request +type AssumeRoleWithLDAPResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithLDAPIdentityResponse" json:"-"` + Result LDAPIdentityResult `xml:"AssumeRoleWithLDAPIdentityResult"` + ResponseMetadata struct { + RequestID string `xml:"RequestId,omitempty"` + } `xml:"ResponseMetadata,omitempty"` +} + +// LDAPIdentityResult - contains credentials for a successful +// AssumeRoleWithLDAPIdentity request. +type LDAPIdentityResult struct { + Credentials struct { + AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"` + SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"` + Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"` + SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"` + } `xml:",omitempty"` + + SubjectFromToken string `xml:",omitempty"` +} + +// LDAPIdentity retrieves credentials from MinIO +type LDAPIdentity struct { + Expiry + + stsEndpoint string + + ldapUsername, ldapPassword string +} + +// NewLDAPIdentity returns new credentials object that uses LDAP +// Identity. +func NewLDAPIdentity(stsEndpoint, ldapUsername, ldapPassword string) (*Credentials, error) { + return New(&LDAPIdentity{ + stsEndpoint: stsEndpoint, + ldapUsername: ldapUsername, + ldapPassword: ldapPassword, + }), nil +} + +// Retrieve gets the credential by calling the MinIO STS API for +// LDAP on the configured stsEndpoint. +func (k *LDAPIdentity) Retrieve() (value Value, err error) { + u, kerr := url.Parse(k.stsEndpoint) + if kerr != nil { + err = kerr + return + } + + clnt := &http.Client{Transport: http.DefaultTransport} + v := url.Values{} + v.Set("Action", "AssumeRoleWithLDAPIdentity") + v.Set("Version", "2011-06-15") + v.Set("LDAPUsername", k.ldapUsername) + v.Set("LDAPPassword", k.ldapPassword) + + u.RawQuery = v.Encode() + + req, kerr := http.NewRequest("POST", u.String(), nil) + if kerr != nil { + err = kerr + return + } + + resp, kerr := clnt.Do(req) + if kerr != nil { + err = kerr + return + } + + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + err = errors.New(resp.Status) + return + } + + r := AssumeRoleWithLDAPResponse{} + if err = xml.NewDecoder(resp.Body).Decode(&r); err != nil { + return + } + + cr := r.Result.Credentials + k.SetExpiration(cr.Expiration, DefaultExpiryWindow) + return Value{ + AccessKeyID: cr.AccessKey, + SecretAccessKey: cr.SecretKey, + SessionToken: cr.SessionToken, + SignerType: SignatureV4, + }, nil +} diff --git a/pkg/credentials/sts_web_identity.go b/pkg/credentials/sts_web_identity.go new file mode 100644 index 0000000..4d53bd2 --- /dev/null +++ b/pkg/credentials/sts_web_identity.go @@ -0,0 +1,158 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "encoding/xml" + "errors" + "fmt" + "net/http" + "net/url" + "time" +) + +// AssumeRoleWithWebIdentityResponse contains the result of successful AssumeRoleWithWebIdentity request. +type AssumeRoleWithWebIdentityResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithWebIdentityResponse" json:"-"` + Result WebIdentityResult `xml:"AssumeRoleWithWebIdentityResult"` + ResponseMetadata struct { + RequestID string `xml:"RequestId,omitempty"` + } `xml:"ResponseMetadata,omitempty"` +} + +// WebIdentityResult - Contains the response to a successful AssumeRoleWithWebIdentity +// request, including temporary credentials that can be used to make MinIO API requests. +type WebIdentityResult struct { + AssumedRoleUser AssumedRoleUser `xml:",omitempty"` + Audience string `xml:",omitempty"` + Credentials struct { + AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"` + SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"` + Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"` + SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"` + } `xml:",omitempty"` + PackedPolicySize int `xml:",omitempty"` + Provider string `xml:",omitempty"` + SubjectFromWebIdentityToken string `xml:",omitempty"` +} + +// WebIdentityToken - web identity token with expiry. +type WebIdentityToken struct { + Token string + Expiry int +} + +// A STSWebIdentity retrieves credentials from MinIO service, and keeps track if +// those credentials are expired. +type STSWebIdentity struct { + Expiry + + // Required http Client to use when connecting to MinIO STS service. + Client *http.Client + + // MinIO endpoint to fetch STS credentials. + stsEndpoint string + + // getWebIDTokenExpiry function which returns ID tokens + // from IDP. This function should return two values one + // is ID token which is a self contained ID token (JWT) + // and second return value is the expiry associated with + // this token. + // This is a customer provided function and is mandatory. + getWebIDTokenExpiry func() (*WebIdentityToken, error) +} + +// NewSTSWebIdentity returns a pointer to a new +// Credentials object wrapping the STSWebIdentity. +func NewSTSWebIdentity(stsEndpoint string, getWebIDTokenExpiry func() (*WebIdentityToken, error)) (*Credentials, error) { + if stsEndpoint == "" { + return nil, errors.New("STS endpoint cannot be empty") + } + if getWebIDTokenExpiry == nil { + return nil, errors.New("Web ID token and expiry retrieval function should be defined") + } + return New(&STSWebIdentity{ + Client: &http.Client{ + Transport: http.DefaultTransport, + }, + stsEndpoint: stsEndpoint, + getWebIDTokenExpiry: getWebIDTokenExpiry, + }), nil +} + +func getWebIdentityCredentials(clnt *http.Client, endpoint string, + getWebIDTokenExpiry func() (*WebIdentityToken, error)) (AssumeRoleWithWebIdentityResponse, error) { + idToken, err := getWebIDTokenExpiry() + if err != nil { + return AssumeRoleWithWebIdentityResponse{}, err + } + + v := url.Values{} + v.Set("Action", "AssumeRoleWithWebIdentity") + v.Set("WebIdentityToken", idToken.Token) + v.Set("DurationSeconds", fmt.Sprintf("%d", idToken.Expiry)) + v.Set("Version", "2011-06-15") + + u, err := url.Parse(endpoint) + if err != nil { + return AssumeRoleWithWebIdentityResponse{}, err + } + + u.RawQuery = v.Encode() + + req, err := http.NewRequest("POST", u.String(), nil) + if err != nil { + return AssumeRoleWithWebIdentityResponse{}, err + } + + resp, err := clnt.Do(req) + if err != nil { + return AssumeRoleWithWebIdentityResponse{}, err + } + + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return AssumeRoleWithWebIdentityResponse{}, errors.New(resp.Status) + } + + a := AssumeRoleWithWebIdentityResponse{} + if err = xml.NewDecoder(resp.Body).Decode(&a); err != nil { + return AssumeRoleWithWebIdentityResponse{}, err + } + + return a, nil +} + +// Retrieve retrieves credentials from the MinIO service. +// Error will be returned if the request fails. +func (m *STSWebIdentity) Retrieve() (Value, error) { + a, err := getWebIdentityCredentials(m.Client, m.stsEndpoint, m.getWebIDTokenExpiry) + if err != nil { + return Value{}, err + } + + // Expiry window is set to 10secs. + m.SetExpiration(a.Result.Credentials.Expiration, DefaultExpiryWindow) + + return Value{ + AccessKeyID: a.Result.Credentials.AccessKey, + SecretAccessKey: a.Result.Credentials.SecretKey, + SessionToken: a.Result.Credentials.SessionToken, + SignerType: SignatureV4, + }, nil +} diff --git a/pkg/encrypt/server-side.go b/pkg/encrypt/server-side.go index 2d3c70f..ac0b69a 100644 --- a/pkg/encrypt/server-side.go +++ b/pkg/encrypt/server-side.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/policy/bucket-policy-condition.go b/pkg/policy/bucket-policy-condition.go index 737b810..b256faf 100644 --- a/pkg/policy/bucket-policy-condition.go +++ b/pkg/policy/bucket-policy-condition.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ package policy -import "github.com/minio/minio-go/pkg/set" +import "github.com/minio/minio-go/v6/pkg/set" // ConditionKeyMap - map of policy condition key and value. type ConditionKeyMap map[string]set.StringSet diff --git a/pkg/policy/bucket-policy-condition_test.go b/pkg/policy/bucket-policy-condition_test.go index 9e4aa8f..2fc9baa 100644 --- a/pkg/policy/bucket-policy-condition_test.go +++ b/pkg/policy/bucket-policy-condition_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ import ( "encoding/json" "testing" - "github.com/minio/minio-go/pkg/set" + "github.com/minio/minio-go/v6/pkg/set" ) // ConditionKeyMap.Add() is called and the result is validated. diff --git a/pkg/policy/bucket-policy.go b/pkg/policy/bucket-policy.go index 79fd801..f2c7289 100644 --- a/pkg/policy/bucket-policy.go +++ b/pkg/policy/bucket-policy.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "reflect" "strings" - "github.com/minio/minio-go/pkg/set" + "github.com/minio/minio-go/v6/pkg/set" ) // BucketPolicy - Bucket level policy. diff --git a/pkg/policy/bucket-policy_test.go b/pkg/policy/bucket-policy_test.go index 1a71d87..a514eaa 100644 --- a/pkg/policy/bucket-policy_test.go +++ b/pkg/policy/bucket-policy_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "reflect" "testing" - "github.com/minio/minio-go/pkg/set" + "github.com/minio/minio-go/v6/pkg/set" ) // TestUnmarshalBucketPolicy tests unmarsheling various examples diff --git a/pkg/s3signer/request-signature-streaming.go b/pkg/s3signer/request-signature-streaming.go index 156a6d6..810b47c 100644 --- a/pkg/s3signer/request-signature-streaming.go +++ b/pkg/s3signer/request-signature-streaming.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -285,7 +285,7 @@ func (s *StreamingReader) Read(buf []byte) (int, error) { // bytes read from baseReader different than // content length provided. if s.bytesRead != s.contentLen { - return 0, io.ErrUnexpectedEOF + return 0, fmt.Errorf("http: ContentLength=%d with Body length %d", s.contentLen, s.bytesRead) } // Sign the chunk and write it to s.buf. diff --git a/pkg/s3signer/request-signature-streaming_test.go b/pkg/s3signer/request-signature-streaming_test.go index 297ab97..e65061e 100644 --- a/pkg/s3signer/request-signature-streaming_test.go +++ b/pkg/s3signer/request-signature-streaming_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/request-signature-v2.go b/pkg/s3signer/request-signature-v2.go index b407093..40ba071 100644 --- a/pkg/s3signer/request-signature-v2.go +++ b/pkg/s3signer/request-signature-v2.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ import ( "strings" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // Signature and API related constants. diff --git a/pkg/s3signer/request-signature-v2_test.go b/pkg/s3signer/request-signature-v2_test.go index 042b6e6..d94e012 100644 --- a/pkg/s3signer/request-signature-v2_test.go +++ b/pkg/s3signer/request-signature-v2_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/request-signature-v4.go b/pkg/s3signer/request-signature-v4.go index daf02fe..ab96b58 100644 --- a/pkg/s3signer/request-signature-v4.go +++ b/pkg/s3signer/request-signature-v4.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ import ( "strings" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // Signature and API related constants. @@ -151,7 +151,7 @@ func getCanonicalHeaders(req http.Request, ignoredHeaders map[string]bool) strin if idx > 0 { buf.WriteByte(',') } - buf.WriteString(v) + buf.WriteString(signV4TrimAll(v)) } buf.WriteByte('\n') } diff --git a/pkg/s3signer/request-signature-v4_test.go b/pkg/s3signer/request-signature-v4_test.go index a109a4f..d0c9e30 100644 --- a/pkg/s3signer/request-signature-v4_test.go +++ b/pkg/s3signer/request-signature-v4_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/request-signature_test.go b/pkg/s3signer/request-signature_test.go index 75115d1..3e072cf 100644 --- a/pkg/s3signer/request-signature_test.go +++ b/pkg/s3signer/request-signature_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/test-utils_test.go b/pkg/s3signer/test-utils_test.go index cf96d66..8a7d8ff 100644 --- a/pkg/s3signer/test-utils_test.go +++ b/pkg/s3signer/test-utils_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3signer/utils.go b/pkg/s3signer/utils.go index 33b1752..934e33a 100644 --- a/pkg/s3signer/utils.go +++ b/pkg/s3signer/utils.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,8 +19,10 @@ package s3signer import ( "crypto/hmac" - "crypto/sha256" "net/http" + "strings" + + "github.com/minio/sha256-simd" ) // unsignedPayload - value to be set to X-Amz-Content-Sha256 header when @@ -47,3 +49,11 @@ func getHostAddr(req *http.Request) string { } return req.URL.Host } + +// Trim leading and trailing spaces and replace sequential spaces with one space, following Trimall() +// in http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html +func signV4TrimAll(input string) string { + // Compress adjacent spaces (a space is determined by + // unicode.IsSpace() internally here) to one space and return + return strings.Join(strings.Fields(input), " ") +} diff --git a/pkg/s3signer/utils_test.go b/pkg/s3signer/utils_test.go index e7fc7b3..5ec0307 100644 --- a/pkg/s3signer/utils_test.go +++ b/pkg/s3signer/utils_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -84,3 +84,32 @@ func TestEncodeURL2Path(t *testing.T) { } } + +// TestSignV4TrimAll - tests the logic of TrimAll() function +func TestSignV4TrimAll(t *testing.T) { + testCases := []struct { + // Input. + inputStr string + // Expected result. + result string + }{ + {"本語", "本語"}, + {" abc ", "abc"}, + {" a b ", "a b"}, + {"a b ", "a b"}, + {"a b", "a b"}, + {"a b", "a b"}, + {" a b c ", "a b c"}, + {"a \t b c ", "a b c"}, + {"\"a \t b c ", "\"a b c"}, + {" \t\n\u000b\r\fa \t\n\u000b\r\f b \t\n\u000b\r\f c \t\n\u000b\r\f", "a b c"}, + } + + // Tests generated values from url encoded name. + for i, testCase := range testCases { + result := signV4TrimAll(testCase.inputStr) + if testCase.result != result { + t.Errorf("Test %d: Expected signV4TrimAll result to be \"%s\", but found it to be \"%s\" instead", i+1, testCase.result, result) + } + } +} diff --git a/pkg/s3utils/utils.go b/pkg/s3utils/utils.go index adceb7f..9af2997 100644 --- a/pkg/s3utils/utils.go +++ b/pkg/s3utils/utils.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -47,8 +47,8 @@ func IsValidDomain(host string) bool { if host[len(host)-1:] == "_" || host[:1] == "_" { return false } - // host cannot start or end with a "." - if host[len(host)-1:] == "." || host[:1] == "." { + // host cannot start with a "." + if host[:1] == "." { return false } // All non alphanumeric characters are invalid. @@ -282,7 +282,7 @@ func checkBucketNameCommon(bucketName string, strict bool) (err error) { if ipAddress.MatchString(bucketName) { return errors.New("Bucket name cannot be an ip address") } - if strings.Contains(bucketName, "..") { + if strings.Contains(bucketName, "..") || strings.Contains(bucketName, ".-") || strings.Contains(bucketName, "-.") { return errors.New("Bucket name contains invalid characters") } if strict { diff --git a/pkg/s3utils/utils_test.go b/pkg/s3utils/utils_test.go index 55eaaea..587b5cd 100644 --- a/pkg/s3utils/utils_test.go +++ b/pkg/s3utils/utils_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -99,6 +99,7 @@ func TestIsValidDomain(t *testing.T) { {"s3.amz.test.com", true}, {"s3.%%", false}, {"localhost", true}, + {"localhost.", true}, // http://www.dns-sd.org/trailingdotsindomainnames.html {"-localhost", false}, {"", false}, {"\n \t", false}, @@ -335,6 +336,8 @@ func TestIsValidBucketName(t *testing.T) { {"my", errors.New("Bucket name cannot be smaller than 3 characters"), false}, {"", errors.New("Bucket name cannot be empty"), false}, {"my..bucket", errors.New("Bucket name contains invalid characters"), false}, + {"my.-bucket", errors.New("Bucket name contains invalid characters"), false}, + {"my-.bucket", errors.New("Bucket name contains invalid characters"), false}, {"192.168.1.168", errors.New("Bucket name cannot be an ip address"), false}, {":bucketname", errors.New("Bucket name contains invalid characters"), false}, {"_bucketName", errors.New("Bucket name contains invalid characters"), false}, @@ -381,6 +384,8 @@ func TestIsValidBucketNameStrict(t *testing.T) { {"my", errors.New("Bucket name cannot be smaller than 3 characters"), false}, {"", errors.New("Bucket name cannot be empty"), false}, {"my..bucket", errors.New("Bucket name contains invalid characters"), false}, + {"my.-bucket", errors.New("Bucket name contains invalid characters"), false}, + {"my-.bucket", errors.New("Bucket name contains invalid characters"), false}, {"192.168.1.168", errors.New("Bucket name cannot be an ip address"), false}, {"Mybucket", errors.New("Bucket name contains invalid characters"), false}, {"my.bucket.com", nil, true}, diff --git a/pkg/set/stringset.go b/pkg/set/stringset.go index efd0262..e220271 100644 --- a/pkg/set/stringset.go +++ b/pkg/set/stringset.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/set/stringset_test.go b/pkg/set/stringset_test.go index d7e6aa7..12aa6f9 100644 --- a/pkg/set/stringset_test.go +++ b/pkg/set/stringset_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -127,7 +127,7 @@ func TestStringSetFuncMatch(t *testing.T) { }{ // Test to check match function doing case insensive compare. {func(setValue string, compareValue string) bool { - return strings.ToUpper(setValue) == strings.ToUpper(compareValue) + return strings.EqualFold(setValue, compareValue) }, "Bar", `[bar]`}, // Test to check match function doing prefix check. {func(setValue string, compareValue string) bool { diff --git a/post-policy.go b/post-policy.go index c285fde..f9250b2 100644 --- a/post-policy.go +++ b/post-policy.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/retry-continous.go b/retry-continous.go index f31dfa6..3d25883 100644 --- a/retry-continous.go +++ b/retry-continous.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -111,6 +111,9 @@ func isHTTPReqErrorRetryable(err error) bool { } else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") { // If error is transport connection broken, retry. return true + } else if strings.Contains(err.Error(), "net/http: timeout awaiting response headers") { + // Retry errors due to server not sending the response before timeout + return true } } return false @@ -139,10 +142,11 @@ func isS3CodeRetryable(s3Code string) (ok bool) { // List of HTTP status codes which are retryable. var retryableHTTPStatusCodes = map[int]struct{}{ - 429: {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet + 429: {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet http.StatusInternalServerError: {}, http.StatusBadGateway: {}, http.StatusServiceUnavailable: {}, + http.StatusGatewayTimeout: {}, // Add more HTTP status codes here. } diff --git a/s3-endpoints.go b/s3-endpoints.go index 0589295..989f58c 100644 --- a/s3-endpoints.go +++ b/s3-endpoints.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,22 +19,25 @@ package minio // awsS3EndpointMap Amazon S3 endpoint map. var awsS3EndpointMap = map[string]string{ - "us-east-1": "s3.amazonaws.com", - "us-east-2": "s3-us-east-2.amazonaws.com", - "us-west-2": "s3-us-west-2.amazonaws.com", - "us-west-1": "s3-us-west-1.amazonaws.com", - "ca-central-1": "s3-ca-central-1.amazonaws.com", - "eu-west-1": "s3-eu-west-1.amazonaws.com", - "eu-west-2": "s3-eu-west-2.amazonaws.com", - "eu-west-3": "s3-eu-west-3.amazonaws.com", - "eu-central-1": "s3-eu-central-1.amazonaws.com", - "ap-south-1": "s3-ap-south-1.amazonaws.com", - "ap-southeast-1": "s3-ap-southeast-1.amazonaws.com", - "ap-southeast-2": "s3-ap-southeast-2.amazonaws.com", - "ap-northeast-1": "s3-ap-northeast-1.amazonaws.com", - "ap-northeast-2": "s3-ap-northeast-2.amazonaws.com", - "sa-east-1": "s3-sa-east-1.amazonaws.com", - "us-gov-west-1": "s3-us-gov-west-1.amazonaws.com", + "us-east-1": "s3.dualstack.us-east-1.amazonaws.com", + "us-east-2": "s3.dualstack.us-east-2.amazonaws.com", + "us-west-2": "s3.dualstack.us-west-2.amazonaws.com", + "us-west-1": "s3.dualstack.us-west-1.amazonaws.com", + "ca-central-1": "s3.dualstack.ca-central-1.amazonaws.com", + "eu-west-1": "s3.dualstack.eu-west-1.amazonaws.com", + "eu-west-2": "s3.dualstack.eu-west-2.amazonaws.com", + "eu-west-3": "s3.dualstack.eu-west-3.amazonaws.com", + "eu-central-1": "s3.dualstack.eu-central-1.amazonaws.com", + "eu-north-1": "s3.dualstack.eu-north-1.amazonaws.com", + "ap-east-1": "s3.dualstack.ap-east-1.amazonaws.com", + "ap-south-1": "s3.dualstack.ap-south-1.amazonaws.com", + "ap-southeast-1": "s3.dualstack.ap-southeast-1.amazonaws.com", + "ap-southeast-2": "s3.dualstack.ap-southeast-2.amazonaws.com", + "ap-northeast-1": "s3.dualstack.ap-northeast-1.amazonaws.com", + "ap-northeast-2": "s3.dualstack.ap-northeast-2.amazonaws.com", + "sa-east-1": "s3.dualstack.sa-east-1.amazonaws.com", + "us-gov-west-1": "s3.dualstack.us-gov-west-1.amazonaws.com", + "us-gov-east-1": "s3.dualstack.us-gov-east-1.amazonaws.com", "cn-north-1": "s3.cn-north-1.amazonaws.com.cn", "cn-northwest-1": "s3.cn-northwest-1.amazonaws.com.cn", } @@ -43,8 +46,8 @@ var awsS3EndpointMap = map[string]string{ func getS3Endpoint(bucketLocation string) (s3Endpoint string) { s3Endpoint, ok := awsS3EndpointMap[bucketLocation] if !ok { - // Default to 's3.amazonaws.com' endpoint. - s3Endpoint = "s3.amazonaws.com" + // Default to 's3.dualstack.us-east-1.amazonaws.com' endpoint. + s3Endpoint = "s3.dualstack.us-east-1.amazonaws.com" } return s3Endpoint } diff --git a/s3-error.go b/s3-error.go index 3b11776..f365157 100644 --- a/s3-error.go +++ b/s3-error.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/staticcheck.conf b/staticcheck.conf new file mode 100644 index 0000000..71cc6f5 --- /dev/null +++ b/staticcheck.conf @@ -0,0 +1 @@ +checks = ["all", "-ST1005", "-ST1017", "-SA9004", "-ST1000", "-S1021"]
\ No newline at end of file diff --git a/test-utils_test.go b/test-utils_test.go index 6f6443c..88b2fac 100644 --- a/test-utils_test.go +++ b/test-utils_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/testcerts/private.key b/testcerts/private.key new file mode 100644 index 0000000..09448cf --- /dev/null +++ b/testcerts/private.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCwUyKC2VOXy2+8 +gMQkRrDJ4aA7K5pgj6LHWu25GeY93x+8DLFyQ9BhoaMcAbs2Cmw91rONDrZ0gNql +yi5JX8t+iiVH8o6dcq6W8jNLnOw0GMNJ2/E1Ckfe5ktkn9synSSwMdnFp3cDk7Hb +2j6IiWrb+PXb7VGL47kDrG59iKQ350MiB3PNpd1ulHbi2m2ZC3WyoTTzlgeTXiXa +zhBIX4wsGVYs6RzS1bTZFBq05dIPNMJCRDVBSBYAAVuBxKjh4xvhC6j0rTCCK8uJ +752KioW4Y0VAEv6yUC4Ht6D9Jcj7gODTgb2irWSCNXFH+pZaI6wWlS8pPiL6iljY +P3kBeFiLAgMBAAECggEAKM20SM9+FryPSPILcdGiC7XY3JiEix/yLWwPYyxpKZw+ +vce6MJUc3dsH4e1Mo37Z+Z17w4LKGj/PWVpmR7iRYOEbK4EoG6t0V54I3NCdoJiy +aJ8rPHj6lMx6WfjcQuQ2n0eJ+8F7OyqsmBHzMqmKPwln69MJcfPq1rzKfOZoCj9p +0oZ+3Iv3roC4uH8peZFooCDUlzJL+8KiybVlemNfklKsHfRmL2vOdFBt+qvit6N/ +9JgBTX1mRx1+vqECj+TlVP//k3BTEPNfpIvsLCRN0eBbQcXYzu/gZfHwGnsy5Lxy +HaHNJnmLZMWSCc4iyCK7uN/BHXNUSSh3qqp4wqz0IQKBgQDdGbOuOVdJW4J1yYua +nDLAu2RQqvZTnoz1/b8jcrjS7tS1la5x1IN0Z9/VqTmkfxyHOK9ab1iVlklkIzjP +CmHnadUwr8vrLcdicFpjVLQU3O4ZqGrgiSGIPAotvOfAOuuzMs+r5ElW/MrGq0Pa +/3tGCTIx8JscZZjGhffUNoIGeQKBgQDMKB+flQB9Ajeo1JM4y3DtHbMJ5D2+/qoe +IkM7kN5K85EEpNwA2PMNKL2qthgM9YFU3K6Dj0gxPNsUKg3W7Ff2r+gaj8K+VjU0 +VbdhTZANbou8hU551swDUCUgquassMtZJIdZnQ7puwLGK67sZwWlOS6Pe1aqaNc5 +nY/MRbemIwKBgEySfykCkNlGCPuUDnZATE91VrudSewRyA3VkGHNdHcQ4bf1m9Gu +YMxqwRl1HxJ6Nz4ZgplWYJ6FyusUS7NgjCGiBIR1DbFoTFoqQROPnUJwdUGLk2Ap +/eP5ryjB+J0ZitGn8kY8rK2kpPGDFN/+hQnvW2PySTXfdbajZP4o1oU5AoGAMiT0 +x3yQlyPRSf2Uf5Gwlf0Ceb5+0Ae6/xXJT7sgbmZuyyY3B1pCMIw+MczyEVTHxHFD +x/qMb9OTt9swdQauAGBqcQO4gImqHcWj+hlT9Yied9qCUPjKOVIZHHH9oJL4D1gi +iodCH3SYlNYr69LOFyv5XLKdsdN4caVaqYDCP+MCgYEAwXyCmSml5oCxJeAOrEDC +Yg3vq3Ul9JO1wc8VDXn9+DtnFsuRHm0fTIxBelmis8AjIIq5DcObpk6wGYZwUiTU +LYQU7v0/Azujv9cl10GI8wzYKiRvExZDTn0sp6OKnau735qBUZvsRDqEQQ5n7waZ +xjlGmZyfah17laYZV9aJoHk= +-----END PRIVATE KEY----- diff --git a/testcerts/public.crt b/testcerts/public.crt new file mode 100644 index 0000000..71f4ccc --- /dev/null +++ b/testcerts/public.crt @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEKjCCApKgAwIBAgIRAPVKnAiFmDti207oQPs2VfUwDQYJKoZIhvcNAQELBQAw +VTEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMRUwEwYDVQQLDAxoYXJz +aGFAcHJ0c2MxHDAaBgNVBAMME21rY2VydCBoYXJzaGFAcHJ0c2MwHhcNMTkwMTA3 +MTE1ODE2WhcNMjkwMTA3MTE1ODE2WjBEMScwJQYDVQQKEx5ta2NlcnQgZGV2ZWxv +cG1lbnQgY2VydGlmaWNhdGUxGTAXBgNVBAsMEGhhcnNoYUBiYWNrc3BhY2UwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCwUyKC2VOXy2+8gMQkRrDJ4aA7 +K5pgj6LHWu25GeY93x+8DLFyQ9BhoaMcAbs2Cmw91rONDrZ0gNqlyi5JX8t+iiVH +8o6dcq6W8jNLnOw0GMNJ2/E1Ckfe5ktkn9synSSwMdnFp3cDk7Hb2j6IiWrb+PXb +7VGL47kDrG59iKQ350MiB3PNpd1ulHbi2m2ZC3WyoTTzlgeTXiXazhBIX4wsGVYs +6RzS1bTZFBq05dIPNMJCRDVBSBYAAVuBxKjh4xvhC6j0rTCCK8uJ752KioW4Y0VA +Ev6yUC4Ht6D9Jcj7gODTgb2irWSCNXFH+pZaI6wWlS8pPiL6iljYP3kBeFiLAgMB +AAGjgYUwgYIwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwG +A1UdEwEB/wQCMAAwHwYDVR0jBBgwFoAUD575sRLoRt9dCxSRqbVctoEHt3MwLAYD +VR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMA0GCSqG +SIb3DQEBCwUAA4IBgQC7qDRDNAHtfGtQs1UmvqWvHPI7qcBQgAibYq/Fox6X9ia1 +weQBfNWEoNOsk97wzbTz81ifXIQ0oV11kWE8EdsbXOf9xeFe9FmDn10d4bGjuMLd ++N3OtGKxLWry2xDYEsVHJZxVxwrf5GK6AJSJj/S837Nil6uRuwjvBVTbxmh1q0nV +x63V8Ag65rLS0fu8msSb64N5UHMCQk6IE+BFHY2gh0lBfZHMdtP4IbeCm756K78/ +WMeqjavGA3bqzVTixCHnJ9S2VLk/oQUS6mL869jM8+tN5VeE6Qsr1/Q5h+NaFCJg +Ed5xjT9mmnc3BLsOHflb1dg+rA90Zz9wphgebXbJhRNuuDRv81dtRPTzM+evGRGM +iRKtiDpog+K0HulfX2g4ZQ1dItEjYz+JYgUFJG+yCvBlNZ/WsTrIVcUCFKaG5rUC +aNqvKrSXfbzKQx7V/TtUAeSfRk7TBRn5qh8Pl+MmQQsB0L9hwTdnqTNn057tghu4 +3/yIIBpzdWPhQ5uv7Vc= +-----END CERTIFICATE----- diff --git a/transport.go b/transport.go index 88700cf..34efa89 100644 --- a/transport.go +++ b/transport.go @@ -1,8 +1,8 @@ // +build go1.7 go1.8 /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2017-2018 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017-2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,31 +20,63 @@ package minio import ( + "crypto/tls" + "crypto/x509" "net" "net/http" "time" + + "golang.org/x/net/http2" ) // DefaultTransport - this default transport is similar to // http.DefaultTransport but with additional param DisableCompression // is set to true to avoid decompressing content with 'gzip' encoding. -var DefaultTransport http.RoundTripper = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 100, - MaxIdleConnsPerHost: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - // Set this value so that the underlying transport round-tripper - // doesn't try to auto decode the body of objects with - // content-encoding set to `gzip`. - // - // Refer: - // https://golang.org/src/net/http/transport.go?h=roundTrip#L1843 - DisableCompression: true, +var DefaultTransport = func(secure bool) (http.RoundTripper, error) { + tr := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 1024, + MaxIdleConnsPerHost: 1024, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + // Set this value so that the underlying transport round-tripper + // doesn't try to auto decode the body of objects with + // content-encoding set to `gzip`. + // + // Refer: + // https://golang.org/src/net/http/transport.go?h=roundTrip#L1843 + DisableCompression: true, + } + + if secure { + rootCAs, _ := x509.SystemCertPool() + if rootCAs == nil { + // In some systems (like Windows) system cert pool is + // not supported or no certificates are present on the + // system - so we create a new cert pool. + rootCAs = x509.NewCertPool() + } + + // Keep TLS config. + tlsConfig := &tls.Config{ + RootCAs: rootCAs, + // Can't use SSLv3 because of POODLE and BEAST + // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher + // Can't use TLSv1.1 because of RC4 cipher usage + MinVersion: tls.VersionTLS12, + } + tr.TLSClientConfig = tlsConfig + + // Because we create a custom TLSClientConfig, we have to opt-in to HTTP/2. + // See https://github.com/golang/go/issues/14275 + if err := http2.ConfigureTransport(tr); err != nil { + return nil, err + } + } + return tr, nil } @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ package minio import ( "crypto/md5" - "crypto/sha256" "encoding/base64" "encoding/hex" "encoding/xml" @@ -32,7 +31,9 @@ import ( "strings" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/sha256-simd" + + "github.com/minio/minio-go/v6/pkg/s3utils" ) // xmlDecoder provide decoded value in xml. @@ -223,13 +224,15 @@ var supportedHeaders = []string{ "content-disposition", "content-language", "x-amz-website-redirect-location", + "x-amz-object-lock-mode", + "x-amz-object-lock-retain-until-date", "expires", // Add more supported headers here. } // isStorageClassHeader returns true if the header is a supported storage class header func isStorageClassHeader(headerKey string) bool { - return strings.ToLower(amzStorageClass) == strings.ToLower(headerKey) + return strings.EqualFold(amzStorageClass, headerKey) } // isStandardHeader returns true if header is a supported header and not a custom header diff --git a/utils_test.go b/utils_test.go index 2e60f77..0ce1638 100644 --- a/utils_test.go +++ b/utils_test.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ import ( "testing" "time" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // Tests signature redacting function used |