From 40ad4a11635eb0491e97c429431e6c95d58b7d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Sipma?= Date: Sat, 8 Dec 2018 10:16:34 +0100 Subject: Import golang-github-minio-minio-go_6.0.11.orig.tar.gz [dgit import orig golang-github-minio-minio-go_6.0.11.orig.tar.gz] --- .gitignore | 3 + .travis.yml | 30 + CONTRIBUTING.md | 23 + LICENSE | 202 + MAINTAINERS.md | 35 + Makefile | 15 + NOTICE | 2 + README.md | 239 + README_zh_CN.md | 245 + api-compose-object.go | 565 ++ api-compose-object_test.go | 155 + api-datatypes.go | 84 + api-error-response.go | 286 + api-error-response_test.go | 294 + api-get-lifecycle.go | 77 + api-get-object-acl.go | 136 + api-get-object-context.go | 26 + api-get-object-file.go | 125 + api-get-object.go | 659 ++ api-get-options.go | 128 + api-get-policy.go | 78 + api-list.go | 720 ++ api-notification.go | 228 + api-presigned.go | 215 + api-put-bucket.go | 306 + api-put-object-common.go | 111 + api-put-object-context.go | 33 + api-put-object-copy.go | 83 + api-put-object-file-context.go | 64 + api-put-object-file.go | 27 + api-put-object-multipart.go | 372 + api-put-object-streaming.go | 417 ++ api-put-object.go | 267 + api-put-object_test.go | 63 + api-remove.go | 303 + api-s3-datatypes.go | 245 + api-select.go | 520 ++ api-stat.go | 185 + api.go | 887 +++ api_unit_test.go | 210 + appveyor.yml | 39 + bucket-cache.go | 221 + bucket-cache_test.go | 351 + bucket-notification.go | 273 + constants.go | 62 + core.go | 153 + core_test.go | 770 +++ docs/API.md | 1630 +++++ docs/checker.go.template | 21 + docs/validator.go | 227 + docs/zh_CN/API.md | 1786 +++++ docs/zh_CN/CONTRIBUTING.md | 22 + examples/minio/listenbucketnotification.go | 61 + examples/s3/bucketexists.go | 52 + examples/s3/composeobject.go | 78 + examples/s3/copyobject.go | 75 + examples/s3/fgetobject-context.go | 54 + examples/s3/fgetobject.go | 46 + examples/s3/fputencrypted-object.go | 59 + examples/s3/fputobject-context.go | 53 + examples/s3/fputobject.go | 48 + examples/s3/get-encrypted-object.go | 69 + examples/s3/getbucketlifecycle.go | 65 + examples/s3/getbucketnotification.go | 56 + examples/s3/getbucketpolicy.go | 64 + examples/s3/getobject-client-encryption.go | 66 + examples/s3/getobject-context.go | 73 + examples/s3/getobject.go | 64 + examples/s3/getobjectacl.go | 53 + examples/s3/listbuckets.go | 49 + examples/s3/listincompleteuploads.go | 58 + examples/s3/listobjects-N.go | 77 + examples/s3/listobjects.go | 58 + examples/s3/listobjectsV2.go | 58 + examples/s3/makebucket.go | 47 + examples/s3/presignedgetobject.go | 54 + examples/s3/presignedheadobject.go | 54 + examples/s3/presignedpostpolicy.go | 60 + examples/s3/presignedputobject.go | 48 + examples/s3/put-encrypted-object.go | 73 + examples/s3/putobject-client-encryption.go | 75 + examples/s3/putobject-context.go | 68 + examples/s3/putobject-getobject-sse.go | 68 + examples/s3/putobject-progress.go | 64 + examples/s3/putobject-s3-accelerate.go | 62 + examples/s3/putobject-streaming.go | 55 + examples/s3/putobject.go | 58 + examples/s3/removeallbucketnotification.go | 50 + examples/s3/removebucket.go | 49 + examples/s3/removeincompleteupload.go | 47 + examples/s3/removeobject.go | 46 + examples/s3/removeobjects.go | 71 + examples/s3/selectobject.go | 73 + examples/s3/setbucketlifecycle.go | 50 + examples/s3/setbucketnotification.go | 86 + examples/s3/setbucketpolicy.go | 51 + examples/s3/statobject.go | 46 + functional_tests.go | 7817 ++++++++++++++++++++++ get-options_test.go | 57 + hook-reader.go | 71 + pkg/credentials/chain.go | 89 + pkg/credentials/chain_test.go | 144 + pkg/credentials/config.json.sample | 17 + pkg/credentials/credentials.go | 175 + pkg/credentials/credentials.sample | 12 + pkg/credentials/credentials_test.go | 73 + pkg/credentials/doc.go | 62 + pkg/credentials/env_aws.go | 71 + pkg/credentials/env_minio.go | 62 + pkg/credentials/env_test.go | 105 + pkg/credentials/file_aws_credentials.go | 120 + pkg/credentials/file_minio_client.go | 133 + pkg/credentials/file_test.go | 189 + pkg/credentials/iam_aws.go | 252 + pkg/credentials/iam_aws_test.go | 243 + pkg/credentials/signature-type.go | 77 + pkg/credentials/static.go | 67 + pkg/credentials/static_test.go | 68 + pkg/encrypt/server-side.go | 195 + pkg/policy/bucket-policy-condition.go | 116 + pkg/policy/bucket-policy-condition_test.go | 290 + pkg/policy/bucket-policy.go | 668 ++ pkg/policy/bucket-policy_test.go | 2030 ++++++ pkg/s3signer/request-signature-streaming.go | 306 + pkg/s3signer/request-signature-streaming_test.go | 109 + pkg/s3signer/request-signature-v2.go | 316 + pkg/s3signer/request-signature-v2_test.go | 36 + pkg/s3signer/request-signature-v4.go | 315 + pkg/s3signer/request-signature-v4_test.go | 50 + pkg/s3signer/request-signature_test.go | 89 + pkg/s3signer/test-utils_test.go | 104 + pkg/s3signer/utils.go | 49 + pkg/s3signer/utils_test.go | 86 + pkg/s3utils/utils.go | 331 + pkg/s3utils/utils_test.go | 408 ++ pkg/set/stringset.go | 197 + pkg/set/stringset_test.go | 348 + post-policy.go | 270 + retry-continous.go | 69 + retry.go | 153 + s3-endpoints.go | 50 + s3-error.go | 61 + test-utils_test.go | 75 + transport.go | 50 + utils.go | 272 + utils_test.go | 378 ++ 146 files changed, 34234 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 MAINTAINERS.md create mode 100644 Makefile create mode 100644 NOTICE create mode 100644 README.md create mode 100644 README_zh_CN.md create mode 100644 api-compose-object.go create mode 100644 api-compose-object_test.go create mode 100644 api-datatypes.go create mode 100644 api-error-response.go create mode 100644 api-error-response_test.go create mode 100644 api-get-lifecycle.go create mode 100644 api-get-object-acl.go create mode 100644 api-get-object-context.go create mode 100644 api-get-object-file.go create mode 100644 api-get-object.go create mode 100644 api-get-options.go create mode 100644 api-get-policy.go create mode 100644 api-list.go create mode 100644 api-notification.go create mode 100644 api-presigned.go create mode 100644 api-put-bucket.go create mode 100644 api-put-object-common.go create mode 100644 api-put-object-context.go create mode 100644 api-put-object-copy.go create mode 100644 api-put-object-file-context.go create mode 100644 api-put-object-file.go create mode 100644 api-put-object-multipart.go create mode 100644 api-put-object-streaming.go create mode 100644 api-put-object.go create mode 100644 api-put-object_test.go create mode 100644 api-remove.go create mode 100644 api-s3-datatypes.go create mode 100644 api-select.go create mode 100644 api-stat.go create mode 100644 api.go create mode 100644 api_unit_test.go create mode 100644 appveyor.yml create mode 100644 bucket-cache.go create mode 100644 bucket-cache_test.go create mode 100644 bucket-notification.go create mode 100644 constants.go create mode 100644 core.go create mode 100644 core_test.go create mode 100644 docs/API.md create mode 100644 docs/checker.go.template create mode 100644 docs/validator.go create mode 100644 docs/zh_CN/API.md create mode 100644 docs/zh_CN/CONTRIBUTING.md create mode 100644 examples/minio/listenbucketnotification.go create mode 100644 examples/s3/bucketexists.go create mode 100644 examples/s3/composeobject.go create mode 100644 examples/s3/copyobject.go create mode 100644 examples/s3/fgetobject-context.go create mode 100644 examples/s3/fgetobject.go create mode 100644 examples/s3/fputencrypted-object.go create mode 100644 examples/s3/fputobject-context.go create mode 100644 examples/s3/fputobject.go create mode 100644 examples/s3/get-encrypted-object.go create mode 100644 examples/s3/getbucketlifecycle.go create mode 100644 examples/s3/getbucketnotification.go create mode 100644 examples/s3/getbucketpolicy.go create mode 100644 examples/s3/getobject-client-encryption.go create mode 100644 examples/s3/getobject-context.go create mode 100644 examples/s3/getobject.go create mode 100644 examples/s3/getobjectacl.go create mode 100644 examples/s3/listbuckets.go create mode 100644 examples/s3/listincompleteuploads.go create mode 100644 examples/s3/listobjects-N.go create mode 100644 examples/s3/listobjects.go create mode 100644 examples/s3/listobjectsV2.go create mode 100644 examples/s3/makebucket.go create mode 100644 examples/s3/presignedgetobject.go create mode 100644 examples/s3/presignedheadobject.go create mode 100644 examples/s3/presignedpostpolicy.go create mode 100644 examples/s3/presignedputobject.go create mode 100644 examples/s3/put-encrypted-object.go create mode 100644 examples/s3/putobject-client-encryption.go create mode 100644 examples/s3/putobject-context.go create mode 100644 examples/s3/putobject-getobject-sse.go create mode 100644 examples/s3/putobject-progress.go create mode 100644 examples/s3/putobject-s3-accelerate.go create mode 100644 examples/s3/putobject-streaming.go create mode 100644 examples/s3/putobject.go create mode 100644 examples/s3/removeallbucketnotification.go create mode 100644 examples/s3/removebucket.go create mode 100644 examples/s3/removeincompleteupload.go create mode 100644 examples/s3/removeobject.go create mode 100644 examples/s3/removeobjects.go create mode 100644 examples/s3/selectobject.go create mode 100644 examples/s3/setbucketlifecycle.go create mode 100644 examples/s3/setbucketnotification.go create mode 100644 examples/s3/setbucketpolicy.go create mode 100644 examples/s3/statobject.go create mode 100644 functional_tests.go create mode 100644 get-options_test.go create mode 100644 hook-reader.go create mode 100644 pkg/credentials/chain.go create mode 100644 pkg/credentials/chain_test.go create mode 100644 pkg/credentials/config.json.sample create mode 100644 pkg/credentials/credentials.go create mode 100644 pkg/credentials/credentials.sample create mode 100644 pkg/credentials/credentials_test.go create mode 100644 pkg/credentials/doc.go create mode 100644 pkg/credentials/env_aws.go create mode 100644 pkg/credentials/env_minio.go create mode 100644 pkg/credentials/env_test.go create mode 100644 pkg/credentials/file_aws_credentials.go create mode 100644 pkg/credentials/file_minio_client.go create mode 100644 pkg/credentials/file_test.go create mode 100644 pkg/credentials/iam_aws.go create mode 100644 pkg/credentials/iam_aws_test.go create mode 100644 pkg/credentials/signature-type.go create mode 100644 pkg/credentials/static.go create mode 100644 pkg/credentials/static_test.go create mode 100644 pkg/encrypt/server-side.go create mode 100644 pkg/policy/bucket-policy-condition.go create mode 100644 pkg/policy/bucket-policy-condition_test.go create mode 100644 pkg/policy/bucket-policy.go create mode 100644 pkg/policy/bucket-policy_test.go create mode 100644 pkg/s3signer/request-signature-streaming.go create mode 100644 pkg/s3signer/request-signature-streaming_test.go create mode 100644 pkg/s3signer/request-signature-v2.go create mode 100644 pkg/s3signer/request-signature-v2_test.go create mode 100644 pkg/s3signer/request-signature-v4.go create mode 100644 pkg/s3signer/request-signature-v4_test.go create mode 100644 pkg/s3signer/request-signature_test.go create mode 100644 pkg/s3signer/test-utils_test.go create mode 100644 pkg/s3signer/utils.go create mode 100644 pkg/s3signer/utils_test.go create mode 100644 pkg/s3utils/utils.go create mode 100644 pkg/s3utils/utils_test.go create mode 100644 pkg/set/stringset.go create mode 100644 pkg/set/stringset_test.go create mode 100644 post-policy.go create mode 100644 retry-continous.go create mode 100644 retry.go create mode 100644 s3-endpoints.go create mode 100644 s3-error.go create mode 100644 test-utils_test.go create mode 100644 transport.go create mode 100644 utils.go create mode 100644 utils_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fa967ab --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*~ +*.test +validator diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..4ae1ead --- /dev/null +++ b/.travis.yml @@ -0,0 +1,30 @@ +sudo: false +language: go + +os: +- linux + +env: +- ARCH=x86_64 +- ARCH=i686 + +go: +- 1.7.4 +- 1.8.x +- 1.9.x +- tip + +matrix: + fast_finish: true + allow_failures: + - go: tip + +addons: + apt: + packages: + - devscripts + +script: +- diff -au <(gofmt -d .) <(printf "") +- diff -au <(licensecheck --check '.go$' --recursive --lines 0 * | grep -v -w 'Apache (v2.0)') <(printf "") +- make diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..8b1ee86 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,23 @@ + +### Developer Guidelines + +``minio-go`` welcomes your contribution. To make the process as seamless as possible, we ask for the following: + +* Go ahead and fork the project and make your changes. We encourage pull requests to discuss code changes. + - Fork it + - Create your feature branch (git checkout -b my-new-feature) + - Commit your changes (git commit -am 'Add some feature') + - Push to the branch (git push origin my-new-feature) + - Create new Pull Request + +* When you're ready to create a pull request, be sure to: + - Have test cases for the new code. If you have questions about how to do it, please ask in your pull request. + - Run `go fmt` + - Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request. + - Make sure `go test -race ./...` and `go build` completes. + NOTE: go test runs functional tests and requires you to have a AWS S3 account. Set them as environment variables + ``ACCESS_KEY`` and ``SECRET_KEY``. To run shorter version of the tests please use ``go test -short -race ./...`` + +* Read [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project + - `minio-go` project is strictly conformant with Golang style + - if you happen to observe offending code, please feel free to send a pull request diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 0000000..1797307 --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1,35 @@ +# For maintainers only + +## Responsibilities + +Please go through this link [Maintainer Responsibility](https://gist.github.com/abperiasamy/f4d9b31d3186bbd26522) + +### Making new releases +Tag and sign your release commit, additionally this step requires you to have access to Minio's trusted private key. +```sh +$ export GNUPGHOME=/media/${USER}/minio/trusted +$ git tag -s 4.0.0 +$ git push +$ git push --tags +``` + +### Update version +Once release has been made update `libraryVersion` constant in `api.go` to next to be released version. + +```sh +$ grep libraryVersion api.go + libraryVersion = "4.0.1" +``` + +Commit your changes +``` +$ git commit -a -m "Update version for next release" --author "Minio Trusted " +``` + +### Announce +Announce new release by adding release notes at https://github.com/minio/minio-go/releases from `trusted@minio.io` account. Release notes requires two sections `highlights` and `changelog`. Highlights is a bulleted list of salient features in this release and Changelog contains list of all commits since the last release. + +To generate `changelog` +```sh +$ git log --no-color --pretty=format:'-%d %s (%cr) <%an>' .. +``` diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bad81ff --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +all: checks + +checks: + @go get -t ./... + @go vet ./... + @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... + @go get github.com/dustin/go-humanize/... + @go get github.com/sirupsen/logrus/... + @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go + @mkdir -p /tmp/examples && for i in $(echo examples/s3/*); do go build -o /tmp/examples/$(basename ${i:0:-3}) ${i}; done + @go get -u github.com/a8m/mark/... + @go get -u github.com/minio/cli/... + @go get -u golang.org/x/tools/cmd/goimports + @go get -u github.com/gernest/wow/... + @go build docs/validator.go && ./validator -m docs/API.md -t docs/checker.go.tpl diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..c521791 --- /dev/null +++ b/NOTICE @@ -0,0 +1,2 @@ +minio-go +Copyright 2015-2017 Minio, Inc. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..ad9d5e6 --- /dev/null +++ b/README.md @@ -0,0 +1,239 @@ +# Minio Go Client SDK for Amazon S3 Compatible Cloud Storage [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Sourcegraph](https://sourcegraph.com/github.com/minio/minio-go/-/badge.svg)](https://sourcegraph.com/github.com/minio/minio-go?badge) [![Apache V2 License](http://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/minio/minio-go/blob/master/LICENSE) + +The Minio Go Client SDK provides simple APIs to access any Amazon S3 compatible object storage. + +This quickstart guide will show you how to install the Minio client SDK, connect to Minio, and provide a walkthrough for a simple file uploader. For a complete list of APIs and examples, please take a look at the [Go Client API Reference](https://docs.minio.io/docs/golang-client-api-reference). + +This document assumes that you have a working [Go development environment](https://docs.minio.io/docs/how-to-install-golang). + +## Download from Github +```sh +go get -u github.com/minio/minio-go +``` + +## Initialize Minio Client +Minio client requires the following four parameters specified to connect to an Amazon S3 compatible object storage. + +| Parameter | Description| +| :--- | :--- | +| endpoint | URL to object storage service. | +| accessKeyID | Access key is the user ID that uniquely identifies your account. | +| secretAccessKey | Secret key is the password to your account. | +| secure | Set this value to 'true' to enable secure (HTTPS) access. | + + +```go +package main + +import ( + "github.com/minio/minio-go" + "log" +) + +func main() { + endpoint := "play.minio.io:9000" + accessKeyID := "Q3AM3UQ867SPQQA43P2F" + secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" + useSSL := true + + // Initialize minio client object. + minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) + if err != nil { + log.Fatalln(err) + } + + log.Printf("%#v\n", minioClient) // minioClient is now setup +} +``` + +## Quick Start Example - File Uploader +This example program connects to an object storage server, creates a bucket and uploads a file to the bucket. + +We will use the Minio server running at [https://play.minio.io:9000](https://play.minio.io:9000) in this example. Feel free to use this service for testing and development. Access credentials shown in this example are open to the public. + +### FileUploader.go +```go +package main + +import ( + "github.com/minio/minio-go" + "log" +) + +func main() { + endpoint := "play.minio.io:9000" + accessKeyID := "Q3AM3UQ867SPQQA43P2F" + secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" + useSSL := true + + // Initialize minio client object. + minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) + if err != nil { + log.Fatalln(err) + } + + // Make a new bucket called mymusic. + bucketName := "mymusic" + location := "us-east-1" + + err = minioClient.MakeBucket(bucketName, location) + if err != nil { + // Check to see if we already own this bucket (which happens if you run this twice) + exists, err := minioClient.BucketExists(bucketName) + if err == nil && exists { + log.Printf("We already own %s\n", bucketName) + } else { + log.Fatalln(err) + } + } else { + log.Printf("Successfully created %s\n", bucketName) + } + + // Upload the zip file + objectName := "golden-oldies.zip" + filePath := "/tmp/golden-oldies.zip" + contentType := "application/zip" + + // Upload the zip file with FPutObject + n, err := minioClient.FPutObject(bucketName, objectName, filePath, minio.PutObjectOptions{ContentType:contentType}) + if err != nil { + log.Fatalln(err) + } + + log.Printf("Successfully uploaded %s of size %d\n", objectName, n) +} +``` + +### Run FileUploader +```sh +go run file-uploader.go +2016/08/13 17:03:28 Successfully created mymusic +2016/08/13 17:03:40 Successfully uploaded golden-oldies.zip of size 16253413 + +mc ls play/mymusic/ +[2016-05-27 16:02:16 PDT] 17MiB golden-oldies.zip +``` + +## API Reference +The full API Reference is available here. + +* [Complete API Reference](https://docs.minio.io/docs/golang-client-api-reference) + +### API Reference : Bucket Operations +* [`MakeBucket`](https://docs.minio.io/docs/golang-client-api-reference#MakeBucket) +* [`ListBuckets`](https://docs.minio.io/docs/golang-client-api-reference#ListBuckets) +* [`BucketExists`](https://docs.minio.io/docs/golang-client-api-reference#BucketExists) +* [`RemoveBucket`](https://docs.minio.io/docs/golang-client-api-reference#RemoveBucket) +* [`ListObjects`](https://docs.minio.io/docs/golang-client-api-reference#ListObjects) +* [`ListObjectsV2`](https://docs.minio.io/docs/golang-client-api-reference#ListObjectsV2) +* [`ListIncompleteUploads`](https://docs.minio.io/docs/golang-client-api-reference#ListIncompleteUploads) + +### API Reference : Bucket policy Operations +* [`SetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketPolicy) +* [`GetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketPolicy) + +### API Reference : Bucket notification Operations +* [`SetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketNotification) +* [`GetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketNotification) +* [`RemoveAllBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#RemoveAllBucketNotification) +* [`ListenBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#ListenBucketNotification) (Minio Extension) + +### API Reference : File Object Operations +* [`FPutObject`](https://docs.minio.io/docs/golang-client-api-reference#FPutObject) +* [`FGetObject`](https://docs.minio.io/docs/golang-client-api-reference#FGetObject) +* [`FPutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FPutObjectWithContext) +* [`FGetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FGetObjectWithContext) + +### API Reference : Object Operations +* [`GetObject`](https://docs.minio.io/docs/golang-client-api-reference#GetObject) +* [`PutObject`](https://docs.minio.io/docs/golang-client-api-reference#PutObject) +* [`GetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#GetObjectWithContext) +* [`PutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#PutObjectWithContext) +* [`PutObjectStreaming`](https://docs.minio.io/docs/golang-client-api-reference#PutObjectStreaming) +* [`StatObject`](https://docs.minio.io/docs/golang-client-api-reference#StatObject) +* [`CopyObject`](https://docs.minio.io/docs/golang-client-api-reference#CopyObject) +* [`RemoveObject`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObject) +* [`RemoveObjects`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObjects) +* [`RemoveIncompleteUpload`](https://docs.minio.io/docs/golang-client-api-reference#RemoveIncompleteUpload) +* [`SelectObjectContent`](https://docs.minio.io/docs/golang-client-api-reference#SelectObjectContent) + + +### API Reference : Presigned Operations +* [`PresignedGetObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedGetObject) +* [`PresignedPutObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedPutObject) +* [`PresignedHeadObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedHeadObject) +* [`PresignedPostPolicy`](https://docs.minio.io/docs/golang-client-api-reference#PresignedPostPolicy) + +### API Reference : Client custom settings +* [`SetAppInfo`](http://docs.minio.io/docs/golang-client-api-reference#SetAppInfo) +* [`SetCustomTransport`](http://docs.minio.io/docs/golang-client-api-reference#SetCustomTransport) +* [`TraceOn`](http://docs.minio.io/docs/golang-client-api-reference#TraceOn) +* [`TraceOff`](http://docs.minio.io/docs/golang-client-api-reference#TraceOff) + +## Full Examples + +### Full Examples : Bucket Operations +* [makebucket.go](https://github.com/minio/minio-go/blob/master/examples/s3/makebucket.go) +* [listbuckets.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbuckets.go) +* [bucketexists.go](https://github.com/minio/minio-go/blob/master/examples/s3/bucketexists.go) +* [removebucket.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucket.go) +* [listobjects.go](https://github.com/minio/minio-go/blob/master/examples/s3/listobjects.go) +* [listobjectsV2.go](https://github.com/minio/minio-go/blob/master/examples/s3/listobjectsV2.go) +* [listincompleteuploads.go](https://github.com/minio/minio-go/blob/master/examples/s3/listincompleteuploads.go) + +### Full Examples : Bucket policy Operations +* [setbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketpolicy.go) +* [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go) +* [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go) + +### Full Examples : Bucket lifecycle Operations +* [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go) +* [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go) + +### Full Examples : Bucket notification Operations +* [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) +* [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) +* [removeallbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeallbucketnotification.go) +* [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (Minio Extension) + +### Full Examples : File Object Operations +* [fputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject.go) +* [fgetobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fgetobject.go) +* [fputobject-context.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject-context.go) +* [fgetobject-context.go](https://github.com/minio/minio-go/blob/master/examples/s3/fgetobject-context.go) + +### Full Examples : Object Operations +* [putobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/putobject.go) +* [getobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/getobject.go) +* [putobject-context.go](https://github.com/minio/minio-go/blob/master/examples/s3/putobject-context.go) +* [getobject-context.go](https://github.com/minio/minio-go/blob/master/examples/s3/getobject-context.go) +* [statobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/statobject.go) +* [copyobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/copyobject.go) +* [removeobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobject.go) +* [removeincompleteupload.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeincompleteupload.go) +* [removeobjects.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobjects.go) + +### Full Examples : Encrypted Object Operations +* [put-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/put-encrypted-object.go) +* [get-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/get-encrypted-object.go) +* [fput-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputencrypted-object.go) + +### Full Examples : Presigned Operations +* [presignedgetobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedgetobject.go) +* [presignedputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedputobject.go) +* [presignedheadobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedheadobject.go) +* [presignedpostpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedpostpolicy.go) + +## Explore Further +* [Complete Documentation](https://docs.minio.io) +* [Minio Go Client SDK API Reference](https://docs.minio.io/docs/golang-client-api-reference) +* [Go Music Player App Full Application Example](https://docs.minio.io/docs/go-music-player-app) + +## Contribute +[Contributors Guide](https://github.com/minio/minio-go/blob/master/CONTRIBUTING.md) + +[![Build Status](https://travis-ci.org/minio/minio-go.svg)](https://travis-ci.org/minio/minio-go) +[![Build status](https://ci.appveyor.com/api/projects/status/1d05e6nvxcelmrak?svg=true)](https://ci.appveyor.com/project/harshavardhana/minio-go) + +## License +This SDK is distributed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0), see [LICENSE](./LICENSE) and [NOTICE](./NOTICE) for more information. diff --git a/README_zh_CN.md b/README_zh_CN.md new file mode 100644 index 0000000..a5acf19 --- /dev/null +++ b/README_zh_CN.md @@ -0,0 +1,245 @@ +# 适用于与Amazon S3兼容云存储的Minio Go SDK [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Sourcegraph](https://sourcegraph.com/github.com/minio/minio-go/-/badge.svg)](https://sourcegraph.com/github.com/minio/minio-go?badge) + +Minio Go Client SDK提供了简单的API来访问任何与Amazon S3兼容的对象存储服务。 + +**支持的云存储:** + +- AWS Signature Version 4 + - Amazon S3 + - Minio + +- AWS Signature Version 2 + - Google Cloud Storage (兼容模式) + - Openstack Swift + Swift3 middleware + - Ceph Object Gateway + - Riak CS + +本文我们将学习如何安装Minio client SDK，连接到Minio，并提供一下文件上传的示例。对于完整的API以及示例，请参考[Go Client API Reference](https://docs.minio.io/docs/golang-client-api-reference)。 + +本文假设你已经有 [Go开发环境](https://docs.minio.io/docs/how-to-install-golang)。 + +## 从Github下载 +```sh +go get -u github.com/minio/minio-go +``` + +## 初始化Minio Client +Minio client需要以下4个参数来连接与Amazon S3兼容的对象存储。 + +| 参数 | 描述| +| :--- | :--- | +| endpoint | 对象存储服务的URL | +| accessKeyID | Access key是唯一标识你的账户的用户ID。 | +| secretAccessKey | Secret key是你账户的密码。 | +| secure | true代表使用HTTPS | + + +```go +package main + +import ( + "github.com/minio/minio-go" + "log" +) + +func main() { + endpoint := "play.minio.io:9000" + accessKeyID := "Q3AM3UQ867SPQQA43P2F" + secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" + useSSL := true + + // 初使化 minio client对象。 + minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) + if err != nil { + log.Fatalln(err) + } + + log.Printf("%#v\n", minioClient) // minioClient初使化成功 +} +``` + +## 示例-文件上传 +本示例连接到一个对象存储服务，创建一个存储桶并上传一个文件到存储桶中。 + +我们在本示例中使用运行在 [https://play.minio.io:9000](https://play.minio.io:9000) 上的Minio服务，你可以用这个服务来开发和测试。示例中的访问凭据是公开的。 + +### FileUploader.go +```go +package main + +import ( + "github.com/minio/minio-go" + "log" +) + +func main() { + endpoint := "play.minio.io:9000" + accessKeyID := "Q3AM3UQ867SPQQA43P2F" + secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" + useSSL := true + + // 初使化minio client对象。 + minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) + if err != nil { + log.Fatalln(err) + } + + // 创建一个叫mymusic的存储桶。 + bucketName := "mymusic" + location := "us-east-1" + + err = minioClient.MakeBucket(bucketName, location) + if err != nil { + // 检查存储桶是否已经存在。 + exists, err := minioClient.BucketExists(bucketName) + if err == nil && exists { + log.Printf("We already own %s\n", bucketName) + } else { + log.Fatalln(err) + } + } + log.Printf("Successfully created %s\n", bucketName) + + // 上传一个zip文件。 + objectName := "golden-oldies.zip" + filePath := "/tmp/golden-oldies.zip" + contentType := "application/zip" + + // 使用FPutObject上传一个zip文件。 + n, err := minioClient.FPutObject(bucketName, objectName, filePath, minio.PutObjectOptions{ContentType:contentType}) + if err != nil { + log.Fatalln(err) + } + + log.Printf("Successfully uploaded %s of size %d\n", objectName, n) +} +``` + +### 运行FileUploader +```sh +go run file-uploader.go +2016/08/13 17:03:28 Successfully created mymusic +2016/08/13 17:03:40 Successfully uploaded golden-oldies.zip of size 16253413 + +mc ls play/mymusic/ +[2016-05-27 16:02:16 PDT] 17MiB golden-oldies.zip +``` + +## API文档 +完整的API文档在这里。 +* [完整API文档](https://docs.minio.io/docs/golang-client-api-reference) + +### API文档 : 操作存储桶 +* [`MakeBucket`](https://docs.minio.io/docs/golang-client-api-reference#MakeBucket) +* [`ListBuckets`](https://docs.minio.io/docs/golang-client-api-reference#ListBuckets) +* [`BucketExists`](https://docs.minio.io/docs/golang-client-api-reference#BucketExists) +* [`RemoveBucket`](https://docs.minio.io/docs/golang-client-api-reference#RemoveBucket) +* [`ListObjects`](https://docs.minio.io/docs/golang-client-api-reference#ListObjects) +* [`ListObjectsV2`](https://docs.minio.io/docs/golang-client-api-reference#ListObjectsV2) +* [`ListIncompleteUploads`](https://docs.minio.io/docs/golang-client-api-reference#ListIncompleteUploads) + +### API文档 : 存储桶策略 +* [`SetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketPolicy) +* [`GetBucketPolicy`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketPolicy) + +### API文档 : 存储桶通知 +* [`SetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#SetBucketNotification) +* [`GetBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#GetBucketNotification) +* [`RemoveAllBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#RemoveAllBucketNotification) +* [`ListenBucketNotification`](https://docs.minio.io/docs/golang-client-api-reference#ListenBucketNotification) (Minio Extension) + +### API文档 : 操作文件对象 +* [`FPutObject`](https://docs.minio.io/docs/golang-client-api-reference#FPutObject) +* [`FGetObject`](https://docs.minio.io/docs/golang-client-api-reference#FPutObject) +* [`FPutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FPutObjectWithContext) +* [`FGetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#FGetObjectWithContext) + +### API文档 : 操作对象 +* [`GetObject`](https://docs.minio.io/docs/golang-client-api-reference#GetObject) +* [`PutObject`](https://docs.minio.io/docs/golang-client-api-reference#PutObject) +* [`GetObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#GetObjectWithContext) +* [`PutObjectWithContext`](https://docs.minio.io/docs/golang-client-api-reference#PutObjectWithContext) +* [`PutObjectStreaming`](https://docs.minio.io/docs/golang-client-api-reference#PutObjectStreaming) +* [`StatObject`](https://docs.minio.io/docs/golang-client-api-reference#StatObject) +* [`CopyObject`](https://docs.minio.io/docs/golang-client-api-reference#CopyObject) +* [`RemoveObject`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObject) +* [`RemoveObjects`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObjects) +* [`RemoveIncompleteUpload`](https://docs.minio.io/docs/golang-client-api-reference#RemoveIncompleteUpload) + +### API文档: 操作加密对象 +* [`GetEncryptedObject`](https://docs.minio.io/docs/golang-client-api-reference#GetEncryptedObject) +* [`PutEncryptedObject`](https://docs.minio.io/docs/golang-client-api-reference#PutEncryptedObject) + +### API文档 : Presigned操作 +* [`PresignedGetObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedGetObject) +* [`PresignedPutObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedPutObject) +* [`PresignedHeadObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedHeadObject) +* [`PresignedPostPolicy`](https://docs.minio.io/docs/golang-client-api-reference#PresignedPostPolicy) + +### API文档 : 客户端自定义设置 +* [`SetAppInfo`](http://docs.minio.io/docs/golang-client-api-reference#SetAppInfo) +* [`SetCustomTransport`](http://docs.minio.io/docs/golang-client-api-reference#SetCustomTransport) +* [`TraceOn`](http://docs.minio.io/docs/golang-client-api-reference#TraceOn) +* [`TraceOff`](http://docs.minio.io/docs/golang-client-api-reference#TraceOff) + +## 完整示例 + +### 完整示例 : 操作存储桶 +* [makebucket.go](https://github.com/minio/minio-go/blob/master/examples/s3/makebucket.go) +* [listbuckets.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbuckets.go) +* [bucketexists.go](https://github.com/minio/minio-go/blob/master/examples/s3/bucketexists.go) +* [removebucket.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucket.go) +* [listobjects.go](https://github.com/minio/minio-go/blob/master/examples/s3/listobjects.go) +* [listobjectsV2.go](https://github.com/minio/minio-go/blob/master/examples/s3/listobjectsV2.go) +* [listincompleteuploads.go](https://github.com/minio/minio-go/blob/master/examples/s3/listincompleteuploads.go) + +### 完整示例 : 存储桶策略 +* [setbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketpolicy.go) +* [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go) +* [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go) + +### 完整示例 : 存储桶通知 +* [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) +* [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) +* [removeallbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeallbucketnotification.go) +* [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (Minio扩展) + +### 完整示例 : 操作文件对象 +* [fputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject.go) +* [fgetobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fgetobject.go) +* [fputobject-context.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject-context.go) +* [fgetobject-context.go](https://github.com/minio/minio-go/blob/master/examples/s3/fgetobject-context.go) + +### 完整示例 : 操作对象 +* [putobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/putobject.go) +* [getobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/getobject.go) +* [putobject-context.go](https://github.com/minio/minio-go/blob/master/examples/s3/putobject-context.go) +* [getobject-context.go](https://github.com/minio/minio-go/blob/master/examples/s3/getobject-context.go) +* [statobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/statobject.go) +* [copyobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/copyobject.go) +* [removeobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobject.go) +* [removeincompleteupload.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeincompleteupload.go) +* [removeobjects.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobjects.go) + +### 完整示例 : 操作加密对象 +* [put-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/put-encrypted-object.go) +* [get-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/get-encrypted-object.go) +* [fput-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputencrypted-object.go) + +### 完整示例 : Presigned操作 +* [presignedgetobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedgetobject.go) +* [presignedputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedputobject.go) +* [presignedheadobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedheadobject.go) +* [presignedpostpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedpostpolicy.go) + +## 了解更多 +* [完整文档](https://docs.minio.io) +* [Minio Go Client SDK API文档](https://docs.minio.io/docs/golang-client-api-reference) +* [Go 音乐播放器完整示例](https://docs.minio.io/docs/go-music-player-app) + +## 贡献 +[贡献指南](https://github.com/minio/minio-go/blob/master/docs/zh_CN/CONTRIBUTING.md) + +[![Build Status](https://travis-ci.org/minio/minio-go.svg)](https://travis-ci.org/minio/minio-go) +[![Build status](https://ci.appveyor.com/api/projects/status/1d05e6nvxcelmrak?svg=true)](https://ci.appveyor.com/project/harshavardhana/minio-go) + diff --git a/api-compose-object.go b/api-compose-object.go new file mode 100644 index 0000000..3ac36c5 --- /dev/null +++ b/api-compose-object.go @@ -0,0 +1,565 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017, 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/pkg/s3utils" +) + +// DestinationInfo - type with information about the object to be +// created via server-side copy requests, using the Compose API. +type DestinationInfo struct { + bucket, object string + encryption encrypt.ServerSide + + // if no user-metadata is provided, it is copied from source + // (when there is only once source object in the compose + // request) + userMetadata map[string]string +} + +// NewDestinationInfo - creates a compose-object/copy-source +// destination info object. +// +// `encSSEC` is the key info for server-side-encryption with customer +// provided key. If it is nil, no encryption is performed. +// +// `userMeta` is the user-metadata key-value pairs to be set on the +// destination. The keys are automatically prefixed with `x-amz-meta-` +// if needed. If nil is passed, and if only a single source (of any +// size) is provided in the ComposeObject call, then metadata from the +// source is copied to the destination. +func NewDestinationInfo(bucket, object string, sse encrypt.ServerSide, userMeta map[string]string) (d DestinationInfo, err error) { + // Input validation. + if err = s3utils.CheckValidBucketName(bucket); err != nil { + return d, err + } + if err = s3utils.CheckValidObjectName(object); err != nil { + return d, err + } + + // Process custom-metadata to remove a `x-amz-meta-` prefix if + // present and validate that keys are distinct (after this + // prefix removal). + m := make(map[string]string) + for k, v := range userMeta { + if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { + k = k[len("x-amz-meta-"):] + } + if _, ok := m[k]; ok { + return d, ErrInvalidArgument(fmt.Sprintf("Cannot add both %s and x-amz-meta-%s keys as custom metadata", k, k)) + } + m[k] = v + } + + return DestinationInfo{ + bucket: bucket, + object: object, + encryption: sse, + userMetadata: m, + }, nil +} + +// getUserMetaHeadersMap - construct appropriate key-value pairs to send +// as headers from metadata map to pass into copy-object request. For +// single part copy-object (i.e. non-multipart object), enable the +// withCopyDirectiveHeader to set the `x-amz-metadata-directive` to +// `REPLACE`, so that metadata headers from the source are not copied +// over. +func (d *DestinationInfo) getUserMetaHeadersMap(withCopyDirectiveHeader bool) map[string]string { + if len(d.userMetadata) == 0 { + return nil + } + r := make(map[string]string) + if withCopyDirectiveHeader { + r["x-amz-metadata-directive"] = "REPLACE" + } + for k, v := range d.userMetadata { + if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) { + r[k] = v + } else { + r["x-amz-meta-"+k] = v + } + } + return r +} + +// SourceInfo - represents a source object to be copied, using +// server-side copying APIs. +type SourceInfo struct { + bucket, object string + start, end int64 + encryption encrypt.ServerSide + // Headers to send with the upload-part-copy request involving + // this source object. + Headers http.Header +} + +// NewSourceInfo - create a compose-object/copy-object source info +// object. +// +// `decryptSSEC` is the decryption key using server-side-encryption +// with customer provided key. It may be nil if the source is not +// encrypted. +func NewSourceInfo(bucket, object string, sse encrypt.ServerSide) SourceInfo { + r := SourceInfo{ + bucket: bucket, + object: object, + start: -1, // range is unspecified by default + encryption: sse, + Headers: make(http.Header), + } + + // Set the source header + r.Headers.Set("x-amz-copy-source", s3utils.EncodePath(bucket+"/"+object)) + return r +} + +// SetRange - Set the start and end offset of the source object to be +// copied. If this method is not called, the whole source object is +// copied. +func (s *SourceInfo) SetRange(start, end int64) error { + if start > end || start < 0 { + return ErrInvalidArgument("start must be non-negative, and start must be at most end.") + } + // Note that 0 <= start <= end + s.start, s.end = start, end + return nil +} + +// SetMatchETagCond - Set ETag match condition. The object is copied +// only if the etag of the source matches the value given here. +func (s *SourceInfo) SetMatchETagCond(etag string) error { + if etag == "" { + return ErrInvalidArgument("ETag cannot be empty.") + } + s.Headers.Set("x-amz-copy-source-if-match", etag) + return nil +} + +// SetMatchETagExceptCond - Set the ETag match exception +// condition. The object is copied only if the etag of the source is +// not the value given here. +func (s *SourceInfo) SetMatchETagExceptCond(etag string) error { + if etag == "" { + return ErrInvalidArgument("ETag cannot be empty.") + } + s.Headers.Set("x-amz-copy-source-if-none-match", etag) + return nil +} + +// SetModifiedSinceCond - Set the modified since condition. +func (s *SourceInfo) SetModifiedSinceCond(modTime time.Time) error { + if modTime.IsZero() { + return ErrInvalidArgument("Input time cannot be 0.") + } + s.Headers.Set("x-amz-copy-source-if-modified-since", modTime.Format(http.TimeFormat)) + return nil +} + +// SetUnmodifiedSinceCond - Set the unmodified since condition. +func (s *SourceInfo) SetUnmodifiedSinceCond(modTime time.Time) error { + if modTime.IsZero() { + return ErrInvalidArgument("Input time cannot be 0.") + } + s.Headers.Set("x-amz-copy-source-if-unmodified-since", modTime.Format(http.TimeFormat)) + return nil +} + +// Helper to fetch size and etag of an object using a StatObject call. +func (s *SourceInfo) getProps(c Client) (size int64, etag string, userMeta map[string]string, err error) { + // Get object info - need size and etag here. Also, decryption + // headers are added to the stat request if given. + var objInfo ObjectInfo + opts := StatObjectOptions{GetObjectOptions{ServerSideEncryption: encrypt.SSE(s.encryption)}} + objInfo, err = c.statObject(context.Background(), s.bucket, s.object, opts) + if err != nil { + err = ErrInvalidArgument(fmt.Sprintf("Could not stat object - %s/%s: %v", s.bucket, s.object, err)) + } else { + size = objInfo.Size + etag = objInfo.ETag + userMeta = make(map[string]string) + for k, v := range objInfo.Metadata { + if strings.HasPrefix(k, "x-amz-meta-") { + if len(v) > 0 { + userMeta[k] = v[0] + } + } + } + } + return +} + +// Low level implementation of CopyObject API, supports only upto 5GiB worth of copy. +func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, + metadata map[string]string) (ObjectInfo, error) { + + // Build headers. + headers := make(http.Header) + + // Set all the metadata headers. + for k, v := range metadata { + headers.Set(k, v) + } + + // Set the source header + headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject)) + + // Send upload-part-copy request + resp, err := c.executeMethod(ctx, "PUT", requestMetadata{ + bucketName: destBucket, + objectName: destObject, + customHeader: headers, + }) + defer closeResponse(resp) + if err != nil { + return ObjectInfo{}, err + } + + // Check if we got an error response. + if resp.StatusCode != http.StatusOK { + return ObjectInfo{}, httpRespToErrorResponse(resp, srcBucket, srcObject) + } + + cpObjRes := copyObjectResult{} + err = xmlDecoder(resp.Body, &cpObjRes) + if err != nil { + return ObjectInfo{}, err + } + + objInfo := ObjectInfo{ + Key: destObject, + ETag: strings.Trim(cpObjRes.ETag, "\""), + LastModified: cpObjRes.LastModified, + } + return objInfo, nil +} + +func (c Client) copyObjectPartDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, + partID int, startOffset int64, length int64, metadata map[string]string) (p CompletePart, err error) { + + headers := make(http.Header) + + // Set source + headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject)) + + if startOffset < 0 { + return p, ErrInvalidArgument("startOffset must be non-negative") + } + + if length >= 0 { + headers.Set("x-amz-copy-source-range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1)) + } + + for k, v := range metadata { + headers.Set(k, v) + } + + queryValues := make(url.Values) + queryValues.Set("partNumber", strconv.Itoa(partID)) + queryValues.Set("uploadId", uploadID) + + resp, err := c.executeMethod(ctx, "PUT", requestMetadata{ + bucketName: destBucket, + objectName: destObject, + customHeader: headers, + queryValues: queryValues, + }) + defer closeResponse(resp) + if err != nil { + return + } + + // Check if we got an error response. + if resp.StatusCode != http.StatusOK { + return p, httpRespToErrorResponse(resp, destBucket, destObject) + } + + // Decode copy-part response on success. + cpObjRes := copyObjectResult{} + err = xmlDecoder(resp.Body, &cpObjRes) + if err != nil { + return p, err + } + p.PartNumber, p.ETag = partID, cpObjRes.ETag + return p, nil +} + +// uploadPartCopy - helper function to create a part in a multipart +// upload via an upload-part-copy request +// https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html +func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID string, partNumber int, + headers http.Header) (p CompletePart, err error) { + + // Build query parameters + urlValues := make(url.Values) + urlValues.Set("partNumber", strconv.Itoa(partNumber)) + urlValues.Set("uploadId", uploadID) + + // Send upload-part-copy request + resp, err := c.executeMethod(ctx, "PUT", requestMetadata{ + bucketName: bucket, + objectName: object, + customHeader: headers, + queryValues: urlValues, + }) + defer closeResponse(resp) + if err != nil { + return p, err + } + + // Check if we got an error response. + if resp.StatusCode != http.StatusOK { + return p, httpRespToErrorResponse(resp, bucket, object) + } + + // Decode copy-part response on success. + cpObjRes := copyObjectResult{} + err = xmlDecoder(resp.Body, &cpObjRes) + if err != nil { + return p, err + } + p.PartNumber, p.ETag = partNumber, cpObjRes.ETag + return p, nil +} + +// ComposeObjectWithProgress - creates an object using server-side copying of +// existing objects. It takes a list of source objects (with optional +// offsets) and concatenates them into a new object using only +// server-side copying operations. Optionally takes progress reader hook +// for applications to look at current progress. +func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error { + if len(srcs) < 1 || len(srcs) > maxPartsCount { + return ErrInvalidArgument("There must be as least one and up to 10000 source objects.") + } + ctx := context.Background() + srcSizes := make([]int64, len(srcs)) + var totalSize, size, totalParts int64 + var srcUserMeta map[string]string + etags := make([]string, len(srcs)) + var err error + for i, src := range srcs { + size, etags[i], srcUserMeta, err = src.getProps(c) + if err != nil { + return err + } + + // Error out if client side encryption is used in this source object when + // more than one source objects are given. + if len(srcs) > 1 && src.Headers.Get("x-amz-meta-x-amz-key") != "" { + return ErrInvalidArgument( + fmt.Sprintf("Client side encryption is used in source object %s/%s", src.bucket, src.object)) + } + + // Check if a segment is specified, and if so, is the + // segment within object bounds? + if src.start != -1 { + // Since range is specified, + // 0 <= src.start <= src.end + // so only invalid case to check is: + if src.end >= size { + return ErrInvalidArgument( + fmt.Sprintf("SourceInfo %d has invalid segment-to-copy [%d, %d] (size is %d)", + i, src.start, src.end, size)) + } + size = src.end - src.start + 1 + } + + // Only the last source may be less than `absMinPartSize` + if size < absMinPartSize && i < len(srcs)-1 { + return ErrInvalidArgument( + fmt.Sprintf("SourceInfo %d is too small (%d) and it is not the last part", i, size)) + } + + // Is data to copy too large? + totalSize += size + if totalSize > maxMultipartPutObjectSize { + return ErrInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> 5TiB)", totalSize)) + } + + // record source size + srcSizes[i] = size + + // calculate parts needed for current source + totalParts += partsRequired(size) + // Do we need more parts than we are allowed? + if totalParts > maxPartsCount { + return ErrInvalidArgument(fmt.Sprintf( + "Your proposed compose object requires more than %d parts", maxPartsCount)) + } + } + + // Single source object case (i.e. when only one source is + // involved, it is being copied wholly and at most 5GiB in + // size, emptyfiles are also supported). + if (totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize) || (totalSize == 0) { + return c.CopyObjectWithProgress(dst, srcs[0], progress) + } + + // Now, handle multipart-copy cases. + + // 1. Ensure that the object has not been changed while + // we are copying data. + for i, src := range srcs { + if src.Headers.Get("x-amz-copy-source-if-match") == "" { + src.SetMatchETagCond(etags[i]) + } + } + + // 2. Initiate a new multipart upload. + + // Set user-metadata on the destination object. If no + // user-metadata is specified, and there is only one source, + // (only) then metadata from source is copied. + userMeta := dst.getUserMetaHeadersMap(false) + metaMap := userMeta + if len(userMeta) == 0 && len(srcs) == 1 { + metaMap = srcUserMeta + } + metaHeaders := make(map[string]string) + for k, v := range metaMap { + metaHeaders[k] = v + } + + uploadID, err := c.newUploadID(ctx, dst.bucket, dst.object, PutObjectOptions{ServerSideEncryption: dst.encryption, UserMetadata: metaHeaders}) + if err != nil { + return err + } + + // 3. Perform copy part uploads + objParts := []CompletePart{} + partIndex := 1 + for i, src := range srcs { + h := src.Headers + if src.encryption != nil { + encrypt.SSECopy(src.encryption).Marshal(h) + } + // Add destination encryption headers + if dst.encryption != nil { + dst.encryption.Marshal(h) + } + + // calculate start/end indices of parts after + // splitting. + startIdx, endIdx := calculateEvenSplits(srcSizes[i], src) + for j, start := range startIdx { + end := endIdx[j] + + // Add (or reset) source range header for + // upload part copy request. + h.Set("x-amz-copy-source-range", + fmt.Sprintf("bytes=%d-%d", start, end)) + + // make upload-part-copy request + complPart, err := c.uploadPartCopy(ctx, dst.bucket, + dst.object, uploadID, partIndex, h) + if err != nil { + return err + } + if progress != nil { + io.CopyN(ioutil.Discard, progress, end-start+1) + } + objParts = append(objParts, complPart) + partIndex++ + } + } + + // 4. Make final complete-multipart request. + _, err = c.completeMultipartUpload(ctx, dst.bucket, dst.object, uploadID, + completeMultipartUpload{Parts: objParts}) + if err != nil { + return err + } + return nil +} + +// ComposeObject - creates an object using server-side copying of +// existing objects. It takes a list of source objects (with optional +// offsets) and concatenates them into a new object using only +// server-side copying operations. +func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error { + return c.ComposeObjectWithProgress(dst, srcs, nil) +} + +// partsRequired is maximum parts possible with +// max part size of ceiling(maxMultipartPutObjectSize / (maxPartsCount - 1)) +func partsRequired(size int64) int64 { + maxPartSize := maxMultipartPutObjectSize / (maxPartsCount - 1) + r := size / int64(maxPartSize) + if size%int64(maxPartSize) > 0 { + r++ + } + return r +} + +// calculateEvenSplits - computes splits for a source and returns +// start and end index slices. Splits happen evenly to be sure that no +// part is less than 5MiB, as that could fail the multipart request if +// it is not the last part. +func calculateEvenSplits(size int64, src SourceInfo) (startIndex, endIndex []int64) { + if size == 0 { + return + } + + reqParts := partsRequired(size) + startIndex = make([]int64, reqParts) + endIndex = make([]int64, reqParts) + // Compute number of required parts `k`, as: + // + // k = ceiling(size / copyPartSize) + // + // Now, distribute the `size` bytes in the source into + // k parts as evenly as possible: + // + // r parts sized (q+1) bytes, and + // (k - r) parts sized q bytes, where + // + // size = q * k + r (by simple division of size by k, + // so that 0 <= r < k) + // + start := src.start + if start == -1 { + start = 0 + } + quot, rem := size/reqParts, size%reqParts + nextStart := start + for j := int64(0); j < reqParts; j++ { + curPartSize := quot + if j < rem { + curPartSize++ + } + + cStart := nextStart + cEnd := cStart + curPartSize - 1 + nextStart = cEnd + 1 + + startIndex[j], endIndex[j] = cStart, cEnd + } + return +} diff --git a/api-compose-object_test.go b/api-compose-object_test.go new file mode 100644 index 0000000..295bbc2 --- /dev/null +++ b/api-compose-object_test.go @@ -0,0 +1,155 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package minio + +import ( + "reflect" + "strings" + "testing" +) + +const ( + gb1 = 1024 * 1024 * 1024 + gb5 = 5 * gb1 + gb5p1 = gb5 + 1 + gb10p1 = 2*gb5 + 1 + gb10p2 = 2*gb5 + 2 +) + +func TestPartsRequired(t *testing.T) { + testCases := []struct { + size, ref int64 + }{ + {0, 0}, + {1, 1}, + {gb5, 10}, + {gb5p1, 10}, + {2 * gb5, 20}, + {gb10p1, 20}, + {gb10p2, 20}, + {gb10p1 + gb10p2, 40}, + {maxMultipartPutObjectSize, 10000}, + } + + for i, testCase := range testCases { + res := partsRequired(testCase.size) + if res != testCase.ref { + t.Errorf("Test %d - output did not match with reference results, Expected %d, got %d", i+1, testCase.ref, res) + } + } +} + +func TestCalculateEvenSplits(t *testing.T) { + + testCases := []struct { + // input size and source object + size int64 + src SourceInfo + + // output part-indexes + starts, ends []int64 + }{ + {0, SourceInfo{start: -1}, nil, nil}, + {1, SourceInfo{start: -1}, []int64{0}, []int64{0}}, + {1, SourceInfo{start: 0}, []int64{0}, []int64{0}}, + + {gb1, SourceInfo{start: -1}, []int64{0, 536870912}, []int64{536870911, 1073741823}}, + {gb5, SourceInfo{start: -1}, + []int64{0, 536870912, 1073741824, 1610612736, 2147483648, 2684354560, + 3221225472, 3758096384, 4294967296, 4831838208}, + []int64{536870911, 1073741823, 1610612735, 2147483647, 2684354559, 3221225471, + 3758096383, 4294967295, 4831838207, 5368709119}, + }, + + // 2 part splits + {gb5p1, SourceInfo{start: -1}, + []int64{0, 536870913, 1073741825, 1610612737, 2147483649, 2684354561, + 3221225473, 3758096385, 4294967297, 4831838209}, + []int64{536870912, 1073741824, 1610612736, 2147483648, 2684354560, 3221225472, + 3758096384, 4294967296, 4831838208, 5368709120}, + }, + {gb5p1, SourceInfo{start: -1}, + []int64{0, 536870913, 1073741825, 1610612737, 2147483649, 2684354561, + 3221225473, 3758096385, 4294967297, 4831838209}, + []int64{536870912, 1073741824, 1610612736, 2147483648, 2684354560, 3221225472, + 3758096384, 4294967296, 4831838208, 5368709120}, + }, + + // 3 part splits + {gb10p1, SourceInfo{start: -1}, + []int64{0, 536870913, 1073741825, 1610612737, 2147483649, 2684354561, + 3221225473, 3758096385, 4294967297, 4831838209, 5368709121, + 5905580033, 6442450945, 6979321857, 7516192769, 8053063681, + 8589934593, 9126805505, 9663676417, 10200547329}, + []int64{536870912, 1073741824, 1610612736, 2147483648, 2684354560, + 3221225472, 3758096384, 4294967296, 4831838208, 5368709120, + 5905580032, 6442450944, 6979321856, 7516192768, 8053063680, + 8589934592, 9126805504, 9663676416, 10200547328, 10737418240}, + }, + {gb10p2, SourceInfo{start: -1}, + []int64{0, 536870913, 1073741826, 1610612738, 2147483650, 2684354562, + 3221225474, 3758096386, 4294967298, 4831838210, 5368709122, + 5905580034, 6442450946, 6979321858, 7516192770, 8053063682, + 8589934594, 9126805506, 9663676418, 10200547330}, + []int64{536870912, 1073741825, 1610612737, 2147483649, 2684354561, + 3221225473, 3758096385, 4294967297, 4831838209, 5368709121, + 5905580033, 6442450945, 6979321857, 7516192769, 8053063681, + 8589934593, 9126805505, 9663676417, 10200547329, 10737418241}, + }, + } + + for i, testCase := range testCases { + resStart, resEnd := calculateEvenSplits(testCase.size, testCase.src) + if !reflect.DeepEqual(testCase.starts, resStart) || !reflect.DeepEqual(testCase.ends, resEnd) { + t.Errorf("Test %d - output did not match with reference results, Expected %d/%d, got %d/%d", i+1, testCase.starts, testCase.ends, resStart, resEnd) + } + } +} + +func TestGetUserMetaHeadersMap(t *testing.T) { + + userMetadata := map[string]string{ + "test": "test", + "x-amz-acl": "public-read-write", + "content-type": "application/binary", + "X-Amz-Storage-Class": "rrs", + "x-amz-grant-write": "test@exo.ch", + } + + destInfo := &DestinationInfo{"bucket", "object", nil, userMetadata} + + r := destInfo.getUserMetaHeadersMap(true) + + i := 0 + + if _, ok := r["x-amz-metadata-directive"]; !ok { + t.Errorf("Test %d - metadata directive was expected but is missing", i) + i++ + } + + for k := range r { + if strings.HasSuffix(k, "test") && !strings.HasPrefix(k, "x-amz-meta-") { + t.Errorf("Test %d - meta %q was expected as an x amz meta", i, k) + i++ + } + + if !strings.HasSuffix(k, "test") && strings.HasPrefix(k, "x-amz-meta-") { + t.Errorf("Test %d - an amz/standard/storageClass Header was expected but got an x amz meta data", i) + i++ + } + } +} diff --git a/api-datatypes.go b/api-datatypes.go new file mode 100644 index 0000000..63fc089 --- /dev/null +++ b/api-datatypes.go @@ -0,0 +1,84 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "net/http" + "time" +) + +// BucketInfo container for bucket metadata. +type BucketInfo struct { + // The name of the bucket. + Name string `json:"name"` + // Date the bucket was created. + CreationDate time.Time `json:"creationDate"` +} + +// ObjectInfo container for object metadata. +type ObjectInfo struct { + // An ETag is optionally set to md5sum of an object. In case of multipart objects, + // ETag is of the form MD5SUM-N where MD5SUM is md5sum of all individual md5sums of + // each parts concatenated into one string. + ETag string `json:"etag"` + + Key string `json:"name"` // Name of the object + LastModified time.Time `json:"lastModified"` // Date and time the object was last modified. + Size int64 `json:"size"` // Size in bytes of the object. + ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data. + + // Collection of additional metadata on the object. + // eg: x-amz-meta-*, content-encoding etc. + Metadata http.Header `json:"metadata" xml:"-"` + + // Owner name. + Owner struct { + DisplayName string `json:"name"` + ID string `json:"id"` + } `json:"owner"` + + // The class of storage used to store the object. + StorageClass string `json:"storageClass"` + + // Error + Err error `json:"-"` +} + +// ObjectMultipartInfo container for multipart object metadata. +type ObjectMultipartInfo struct { + // Date and time at which the multipart upload was initiated. + Initiated time.Time `type:"timestamp" timestampFormat:"iso8601"` + + Initiator initiator + Owner owner + + // The type of storage to use for the object. Defaults to 'STANDARD'. + StorageClass string + + // Key of the object for which the multipart upload was initiated. + Key string + + // Size in bytes of the object. + Size int64 + + // Upload ID that identifies the multipart upload. + UploadID string `xml:"UploadId"` + + // Error + Err error +} diff --git a/api-error-response.go b/api-error-response.go new file mode 100644 index 0000000..655991c --- /dev/null +++ b/api-error-response.go @@ -0,0 +1,286 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "encoding/xml" + "fmt" + "net/http" +) + +/* **** SAMPLE ERROR RESPONSE **** + + + AccessDenied + Access Denied + bucketName + objectName + F19772218238A85A + GuWkjyviSiGHizehqpmsD1ndz5NClSP19DOT+s2mv7gXGQ8/X1lhbDGiIJEXpGFD + +*/ + +// ErrorResponse - Is the typed error returned by all API operations. +type ErrorResponse struct { + XMLName xml.Name `xml:"Error" json:"-"` + Code string + Message string + BucketName string + Key string + RequestID string `xml:"RequestId"` + HostID string `xml:"HostId"` + + // Region where the bucket is located. This header is returned + // only in HEAD bucket and ListObjects response. + Region string + + // Underlying HTTP status code for the returned error + StatusCode int `xml:"-" json:"-"` + + // Headers of the returned S3 XML error + Headers http.Header `xml:"-" json:"-"` +} + +// ToErrorResponse - Returns parsed ErrorResponse struct from body and +// http headers. +// +// For example: +// +// import s3 "github.com/minio/minio-go" +// ... +// ... +// reader, stat, err := s3.GetObject(...) +// if err != nil { +// resp := s3.ToErrorResponse(err) +// } +// ... +func ToErrorResponse(err error) ErrorResponse { + switch err := err.(type) { + case ErrorResponse: + return err + default: + return ErrorResponse{} + } +} + +// Error - Returns S3 error string. +func (e ErrorResponse) Error() string { + if e.Message == "" { + msg, ok := s3ErrorResponseMap[e.Code] + if !ok { + msg = fmt.Sprintf("Error response code %s.", e.Code) + } + return msg + } + return e.Message +} + +// Common string for errors to report issue location in unexpected +// cases. +const ( + reportIssue = "Please report this issue at https://github.com/minio/minio-go/issues." +) + +// httpRespToErrorResponse returns a new encoded ErrorResponse +// structure as error. +func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) error { + if resp == nil { + msg := "Response is empty. " + reportIssue + return ErrInvalidArgument(msg) + } + + errResp := ErrorResponse{ + StatusCode: resp.StatusCode, + } + + err := xmlDecoder(resp.Body, &errResp) + // Xml decoding failed with no body, fall back to HTTP headers. + if err != nil { + switch resp.StatusCode { + case http.StatusNotFound: + if objectName == "" { + errResp = ErrorResponse{ + StatusCode: resp.StatusCode, + Code: "NoSuchBucket", + Message: "The specified bucket does not exist.", + BucketName: bucketName, + } + } else { + errResp = ErrorResponse{ + StatusCode: resp.StatusCode, + Code: "NoSuchKey", + Message: "The specified key does not exist.", + BucketName: bucketName, + Key: objectName, + } + } + case http.StatusForbidden: + errResp = ErrorResponse{ + StatusCode: resp.StatusCode, + Code: "AccessDenied", + Message: "Access Denied.", + BucketName: bucketName, + Key: objectName, + } + case http.StatusConflict: + errResp = ErrorResponse{ + StatusCode: resp.StatusCode, + Code: "Conflict", + Message: "Bucket not empty.", + BucketName: bucketName, + } + case http.StatusPreconditionFailed: + errResp = ErrorResponse{ + StatusCode: resp.StatusCode, + Code: "PreconditionFailed", + Message: s3ErrorResponseMap["PreconditionFailed"], + BucketName: bucketName, + Key: objectName, + } + default: + errResp = ErrorResponse{ + StatusCode: resp.StatusCode, + Code: resp.Status, + Message: resp.Status, + BucketName: bucketName, + } + } + } + + // Save hostID, requestID and region information + // from headers if not available through error XML. + if errResp.RequestID == "" { + errResp.RequestID = resp.Header.Get("x-amz-request-id") + } + if errResp.HostID == "" { + errResp.HostID = resp.Header.Get("x-amz-id-2") + } + if errResp.Region == "" { + errResp.Region = resp.Header.Get("x-amz-bucket-region") + } + if errResp.Code == "InvalidRegion" && errResp.Region != "" { + errResp.Message = fmt.Sprintf("Region does not match, expecting region ‘%s’.", errResp.Region) + } + + // Save headers returned in the API XML error + errResp.Headers = resp.Header + + return errResp +} + +// ErrTransferAccelerationBucket - bucket name is invalid to be used with transfer acceleration. +func ErrTransferAccelerationBucket(bucketName string) error { + return ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: "InvalidArgument", + Message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’.", + BucketName: bucketName, + } +} + +// ErrEntityTooLarge - Input size is larger than supported maximum. +func ErrEntityTooLarge(totalSize, maxObjectSize int64, bucketName, objectName string) error { + msg := fmt.Sprintf("Your proposed upload size ‘%d’ exceeds the maximum allowed object size ‘%d’ for single PUT operation.", totalSize, maxObjectSize) + return ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: "EntityTooLarge", + Message: msg, + BucketName: bucketName, + Key: objectName, + } +} + +// ErrEntityTooSmall - Input size is smaller than supported minimum. +func ErrEntityTooSmall(totalSize int64, bucketName, objectName string) error { + msg := fmt.Sprintf("Your proposed upload size ‘%d’ is below the minimum allowed object size ‘0B’ for single PUT operation.", totalSize) + return ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: "EntityTooSmall", + Message: msg, + BucketName: bucketName, + Key: objectName, + } +} + +// ErrUnexpectedEOF - Unexpected end of file reached. +func ErrUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string) error { + msg := fmt.Sprintf("Data read ‘%d’ is not equal to the size ‘%d’ of the input Reader.", totalRead, totalSize) + return ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: "UnexpectedEOF", + Message: msg, + BucketName: bucketName, + Key: objectName, + } +} + +// ErrInvalidBucketName - Invalid bucket name response. +func ErrInvalidBucketName(message string) error { + return ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: "InvalidBucketName", + Message: message, + RequestID: "minio", + } +} + +// ErrInvalidObjectName - Invalid object name response. +func ErrInvalidObjectName(message string) error { + return ErrorResponse{ + StatusCode: http.StatusNotFound, + Code: "NoSuchKey", + Message: message, + RequestID: "minio", + } +} + +// ErrInvalidObjectPrefix - Invalid object prefix response is +// similar to object name response. +var ErrInvalidObjectPrefix = ErrInvalidObjectName + +// ErrInvalidArgument - Invalid argument response. +func ErrInvalidArgument(message string) error { + return ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: "InvalidArgument", + Message: message, + RequestID: "minio", + } +} + +// ErrNoSuchBucketPolicy - No Such Bucket Policy response +// The specified bucket does not have a bucket policy. +func ErrNoSuchBucketPolicy(message string) error { + return ErrorResponse{ + StatusCode: http.StatusNotFound, + Code: "NoSuchBucketPolicy", + Message: message, + RequestID: "minio", + } +} + +// ErrAPINotSupported - API not supported response +// The specified API call is not supported +func ErrAPINotSupported(message string) error { + return ErrorResponse{ + StatusCode: http.StatusNotImplemented, + Code: "APINotSupported", + Message: message, + RequestID: "minio", + } +} diff --git a/api-error-response_test.go b/api-error-response_test.go new file mode 100644 index 0000000..bf10941 --- /dev/null +++ b/api-error-response_test.go @@ -0,0 +1,294 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "encoding/xml" + "fmt" + "io/ioutil" + "net/http" + "reflect" + "strconv" + "testing" +) + +// Tests validate the Error generator function for http response with error. +func TestHttpRespToErrorResponse(t *testing.T) { + // 'genAPIErrorResponse' generates ErrorResponse for given APIError. + // provides a encodable populated response values. + genAPIErrorResponse := func(err APIError, bucketName string) ErrorResponse { + return ErrorResponse{ + Code: err.Code, + Message: err.Description, + BucketName: bucketName, + } + } + + // Encodes the response headers into XML format. + encodeErr := func(response ErrorResponse) []byte { + buf := &bytes.Buffer{} + buf.WriteString(xml.Header) + encoder := xml.NewEncoder(buf) + err := encoder.Encode(response) + if err != nil { + t.Fatalf("error encoding response: %v", err) + } + return buf.Bytes() + } + + // `createAPIErrorResponse` Mocks XML error response from the server. + createAPIErrorResponse := func(APIErr APIError, bucketName string) *http.Response { + // generate error response. + // response body contains the XML error message. + resp := &http.Response{} + errorResponse := genAPIErrorResponse(APIErr, bucketName) + encodedErrorResponse := encodeErr(errorResponse) + // write Header. + resp.StatusCode = APIErr.HTTPStatusCode + resp.Body = ioutil.NopCloser(bytes.NewBuffer(encodedErrorResponse)) + + return resp + } + + // 'genErrResponse' contructs error response based http Status Code + genErrResponse := func(resp *http.Response, code, message, bucketName, objectName string) ErrorResponse { + errResp := ErrorResponse{ + StatusCode: resp.StatusCode, + Code: code, + Message: message, + BucketName: bucketName, + Key: objectName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), + Headers: resp.Header, + } + return errResp + } + + // Generate invalid argument error. + genInvalidError := func(message string) error { + errResp := ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: "InvalidArgument", + Message: message, + RequestID: "minio", + } + return errResp + } + + // Set common http response headers. + setCommonHeaders := func(resp *http.Response) *http.Response { + // set headers. + resp.Header = make(http.Header) + resp.Header.Set("x-amz-request-id", "xyz") + resp.Header.Set("x-amz-id-2", "abc") + resp.Header.Set("x-amz-bucket-region", "us-east-1") + return resp + } + + // Generate http response with empty body. + // Set the StatusCode to the argument supplied. + // Sets common headers. + genEmptyBodyResponse := func(statusCode int) *http.Response { + resp := &http.Response{ + StatusCode: statusCode, + Body: ioutil.NopCloser(bytes.NewReader(nil)), + } + setCommonHeaders(resp) + return resp + } + + // Decode XML error message from the http response body. + decodeXMLError := func(resp *http.Response) error { + errResp := ErrorResponse{ + StatusCode: resp.StatusCode, + } + err := xmlDecoder(resp.Body, &errResp) + if err != nil { + t.Fatalf("XML decoding of response body failed: %v", err) + } + return errResp + } + + // List of APIErrors used to generate/mock server side XML error response. + APIErrors := []APIError{ + { + Code: "NoSuchBucketPolicy", + Description: "The specified bucket does not have a bucket policy.", + HTTPStatusCode: http.StatusNotFound, + }, + } + + // List of expected response. + // Used for asserting the actual response. + expectedErrResponse := []error{ + genInvalidError("Response is empty. " + "Please report this issue at https://github.com/minio/minio-go/issues."), + decodeXMLError(createAPIErrorResponse(APIErrors[0], "minio-bucket")), + genErrResponse(setCommonHeaders(&http.Response{StatusCode: http.StatusNotFound}), "NoSuchBucket", "The specified bucket does not exist.", "minio-bucket", ""), + genErrResponse(setCommonHeaders(&http.Response{StatusCode: http.StatusNotFound}), "NoSuchKey", "The specified key does not exist.", "minio-bucket", "Asia/"), + genErrResponse(setCommonHeaders(&http.Response{StatusCode: http.StatusForbidden}), "AccessDenied", "Access Denied.", "minio-bucket", ""), + genErrResponse(setCommonHeaders(&http.Response{StatusCode: http.StatusConflict}), "Conflict", "Bucket not empty.", "minio-bucket", ""), + genErrResponse(setCommonHeaders(&http.Response{StatusCode: http.StatusBadRequest}), "Bad Request", "Bad Request", "minio-bucket", ""), + } + + // List of http response to be used as input. + inputResponses := []*http.Response{ + nil, + createAPIErrorResponse(APIErrors[0], "minio-bucket"), + genEmptyBodyResponse(http.StatusNotFound), + genEmptyBodyResponse(http.StatusNotFound), + genEmptyBodyResponse(http.StatusForbidden), + genEmptyBodyResponse(http.StatusConflict), + genEmptyBodyResponse(http.StatusBadRequest), + } + + testCases := []struct { + bucketName string + objectName string + inputHTTPResp *http.Response + // expected results. + expectedResult error + // flag indicating whether tests should pass. + + }{ + {"minio-bucket", "", inputResponses[0], expectedErrResponse[0]}, + {"minio-bucket", "", inputResponses[1], expectedErrResponse[1]}, + {"minio-bucket", "", inputResponses[2], expectedErrResponse[2]}, + {"minio-bucket", "Asia/", inputResponses[3], expectedErrResponse[3]}, + {"minio-bucket", "", inputResponses[4], expectedErrResponse[4]}, + {"minio-bucket", "", inputResponses[5], expectedErrResponse[5]}, + } + + for i, testCase := range testCases { + actualResult := httpRespToErrorResponse(testCase.inputHTTPResp, testCase.bucketName, testCase.objectName) + if !reflect.DeepEqual(testCase.expectedResult, actualResult) { + t.Errorf("Test %d: Expected result to be '%#v', but instead got '%#v'", i+1, testCase.expectedResult, actualResult) + } + } +} + +// Test validates 'ErrEntityTooLarge' error response. +func TestErrEntityTooLarge(t *testing.T) { + msg := fmt.Sprintf("Your proposed upload size ‘%d’ exceeds the maximum allowed object size ‘%d’ for single PUT operation.", 1000000, 99999) + expectedResult := ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: "EntityTooLarge", + Message: msg, + BucketName: "minio-bucket", + Key: "Asia/", + } + actualResult := ErrEntityTooLarge(1000000, 99999, "minio-bucket", "Asia/") + if !reflect.DeepEqual(expectedResult, actualResult) { + t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) + } +} + +// Test validates 'ErrEntityTooSmall' error response. +func TestErrEntityTooSmall(t *testing.T) { + msg := fmt.Sprintf("Your proposed upload size ‘%d’ is below the minimum allowed object size ‘0B’ for single PUT operation.", -1) + expectedResult := ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: "EntityTooSmall", + Message: msg, + BucketName: "minio-bucket", + Key: "Asia/", + } + actualResult := ErrEntityTooSmall(-1, "minio-bucket", "Asia/") + if !reflect.DeepEqual(expectedResult, actualResult) { + t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) + } +} + +// Test validates 'ErrUnexpectedEOF' error response. +func TestErrUnexpectedEOF(t *testing.T) { + msg := fmt.Sprintf("Data read ‘%s’ is not equal to the size ‘%s’ of the input Reader.", + strconv.FormatInt(100, 10), strconv.FormatInt(101, 10)) + expectedResult := ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: "UnexpectedEOF", + Message: msg, + BucketName: "minio-bucket", + Key: "Asia/", + } + actualResult := ErrUnexpectedEOF(100, 101, "minio-bucket", "Asia/") + if !reflect.DeepEqual(expectedResult, actualResult) { + t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) + } +} + +// Test validates 'ErrInvalidBucketName' error response. +func TestErrInvalidBucketName(t *testing.T) { + expectedResult := ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: "InvalidBucketName", + Message: "Invalid Bucket name", + RequestID: "minio", + } + actualResult := ErrInvalidBucketName("Invalid Bucket name") + if !reflect.DeepEqual(expectedResult, actualResult) { + t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) + } +} + +// Test validates 'ErrInvalidObjectName' error response. +func TestErrInvalidObjectName(t *testing.T) { + expectedResult := ErrorResponse{ + StatusCode: http.StatusNotFound, + Code: "NoSuchKey", + Message: "Invalid Object Key", + RequestID: "minio", + } + actualResult := ErrInvalidObjectName("Invalid Object Key") + if !reflect.DeepEqual(expectedResult, actualResult) { + t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) + } +} + +// Test validates 'ErrInvalidArgument' response. +func TestErrInvalidArgument(t *testing.T) { + expectedResult := ErrorResponse{ + StatusCode: http.StatusBadRequest, + Code: "InvalidArgument", + Message: "Invalid Argument", + RequestID: "minio", + } + actualResult := ErrInvalidArgument("Invalid Argument") + if !reflect.DeepEqual(expectedResult, actualResult) { + t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) + } +} + +// Tests if the Message field is missing. +func TestErrWithoutMessage(t *testing.T) { + errResp := ErrorResponse{ + Code: "AccessDenied", + RequestID: "minio", + } + if errResp.Error() != "Access Denied." { + t.Errorf("Expected \"Access Denied.\", got %s", errResp) + } + errResp = ErrorResponse{ + Code: "InvalidArgument", + RequestID: "minio", + } + if errResp.Error() != "Error response code InvalidArgument." { + t.Errorf("Expected \"Error response code InvalidArgument.\", got %s", errResp) + } +} diff --git a/api-get-lifecycle.go b/api-get-lifecycle.go new file mode 100644 index 0000000..8097bfc --- /dev/null +++ b/api-get-lifecycle.go @@ -0,0 +1,77 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "io/ioutil" + "net/http" + "net/url" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// GetBucketLifecycle - get bucket lifecycle. +func (c Client) GetBucketLifecycle(bucketName string) (string, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return "", err + } + bucketLifecycle, err := c.getBucketLifecycle(bucketName) + if err != nil { + errResponse := ToErrorResponse(err) + if errResponse.Code == "NoSuchLifecycleConfiguration" { + return "", nil + } + return "", err + } + return bucketLifecycle, nil +} + +// Request server for current bucket lifecycle. +func (c Client) getBucketLifecycle(bucketName string) (string, error) { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("lifecycle", "") + + // Execute GET on bucket to get lifecycle. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + }) + + defer closeResponse(resp) + if err != nil { + return "", err + } + + if resp != nil { + if resp.StatusCode != http.StatusOK { + return "", httpRespToErrorResponse(resp, bucketName, "") + } + } + + bucketLifecycleBuf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + lifecycle := string(bucketLifecycleBuf) + return lifecycle, err +} diff --git a/api-get-object-acl.go b/api-get-object-acl.go new file mode 100644 index 0000000..af5544d --- /dev/null +++ b/api-get-object-acl.go @@ -0,0 +1,136 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "net/http" + "net/url" +) + +type accessControlPolicy struct { + Owner struct { + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` + } `xml:"Owner"` + AccessControlList struct { + Grant []struct { + Grantee struct { + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` + URI string `xml:"URI"` + } `xml:"Grantee"` + Permission string `xml:"Permission"` + } `xml:"Grant"` + } `xml:"AccessControlList"` +} + +//GetObjectACL get object ACLs +func (c Client) GetObjectACL(bucketName, objectName string) (*ObjectInfo, error) { + + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: url.Values{ + "acl": []string{""}, + }, + }) + if err != nil { + return nil, err + } + defer closeResponse(resp) + + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, objectName) + } + + res := &accessControlPolicy{} + + if err := xmlDecoder(resp.Body, res); err != nil { + return nil, err + } + + objInfo, err := c.statObject(context.Background(), bucketName, objectName, StatObjectOptions{}) + if err != nil { + return nil, err + } + + cannedACL := getCannedACL(res) + if cannedACL != "" { + objInfo.Metadata.Add("X-Amz-Acl", cannedACL) + return &objInfo, nil + } + + grantACL := getAmzGrantACL(res) + for k, v := range grantACL { + objInfo.Metadata[k] = v + } + + return &objInfo, nil +} + +func getCannedACL(aCPolicy *accessControlPolicy) string { + grants := aCPolicy.AccessControlList.Grant + + switch { + case len(grants) == 1: + if grants[0].Grantee.URI == "" && grants[0].Permission == "FULL_CONTROL" { + return "private" + } + case len(grants) == 2: + for _, g := range grants { + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && g.Permission == "READ" { + return "authenticated-read" + } + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "READ" { + return "public-read" + } + if g.Permission == "READ" && g.Grantee.ID == aCPolicy.Owner.ID { + return "bucket-owner-read" + } + } + case len(grants) == 3: + for _, g := range grants { + if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "WRITE" { + return "public-read-write" + } + } + } + return "" +} + +func getAmzGrantACL(aCPolicy *accessControlPolicy) map[string][]string { + grants := aCPolicy.AccessControlList.Grant + res := map[string][]string{} + + for _, g := range grants { + switch { + case g.Permission == "READ": + res["X-Amz-Grant-Read"] = append(res["X-Amz-Grant-Read"], "id="+g.Grantee.ID) + case g.Permission == "WRITE": + res["X-Amz-Grant-Write"] = append(res["X-Amz-Grant-Write"], "id="+g.Grantee.ID) + case g.Permission == "READ_ACP": + res["X-Amz-Grant-Read-Acp"] = append(res["X-Amz-Grant-Read-Acp"], "id="+g.Grantee.ID) + case g.Permission == "WRITE_ACP": + res["X-Amz-Grant-Write-Acp"] = append(res["X-Amz-Grant-Write-Acp"], "id="+g.Grantee.ID) + case g.Permission == "FULL_CONTROL": + res["X-Amz-Grant-Full-Control"] = append(res["X-Amz-Grant-Full-Control"], "id="+g.Grantee.ID) + } + } + return res +} diff --git a/api-get-object-context.go b/api-get-object-context.go new file mode 100644 index 0000000..f8dfac7 --- /dev/null +++ b/api-get-object-context.go @@ -0,0 +1,26 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import "context" + +// GetObjectWithContext - returns an seekable, readable object. +// The options can be used to specify the GET request further. +func (c Client) GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) { + return c.getObjectWithContext(ctx, bucketName, objectName, opts) +} diff --git a/api-get-object-file.go b/api-get-object-file.go new file mode 100644 index 0000000..a852220 --- /dev/null +++ b/api-get-object-file.go @@ -0,0 +1,125 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "io" + "os" + "path/filepath" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// FGetObjectWithContext - download contents of an object to a local file. +// The options can be used to specify the GET request further. +func (c Client) FGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error { + return c.fGetObjectWithContext(ctx, bucketName, objectName, filePath, opts) +} + +// FGetObject - download contents of an object to a local file. +func (c Client) FGetObject(bucketName, objectName, filePath string, opts GetObjectOptions) error { + return c.fGetObjectWithContext(context.Background(), bucketName, objectName, filePath, opts) +} + +// fGetObjectWithContext - fgetObject wrapper function with context +func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return err + } + + // Verify if destination already exists. + st, err := os.Stat(filePath) + if err == nil { + // If the destination exists and is a directory. + if st.IsDir() { + return ErrInvalidArgument("fileName is a directory.") + } + } + + // Proceed if file does not exist. return for all other errors. + if err != nil { + if !os.IsNotExist(err) { + return err + } + } + + // Extract top level directory. + objectDir, _ := filepath.Split(filePath) + if objectDir != "" { + // Create any missing top level directories. + if err := os.MkdirAll(objectDir, 0700); err != nil { + return err + } + } + + // Gather md5sum. + objectStat, err := c.StatObject(bucketName, objectName, StatObjectOptions{opts}) + if err != nil { + return err + } + + // Write to a temporary file "fileName.part.minio" before saving. + filePartPath := filePath + objectStat.ETag + ".part.minio" + + // If exists, open in append mode. If not create it as a part file. + filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + return err + } + + // Issue Stat to get the current offset. + st, err = filePart.Stat() + if err != nil { + return err + } + + // Initialize get object request headers to set the + // appropriate range offsets to read from. + if st.Size() > 0 { + opts.SetRange(st.Size(), 0) + } + + // Seek to current position for incoming reader. + objectReader, objectStat, err := c.getObject(ctx, bucketName, objectName, opts) + if err != nil { + return err + } + + // Write to the part file. + if _, err = io.CopyN(filePart, objectReader, objectStat.Size); err != nil { + return err + } + + // Close the file before rename, this is specifically needed for Windows users. + if err = filePart.Close(); err != nil { + return err + } + + // Safely completed. Now commit by renaming to actual filename. + if err = os.Rename(filePartPath, filePath); err != nil { + return err + } + + // Return. + return nil +} diff --git a/api-get-object.go b/api-get-object.go new file mode 100644 index 0000000..0bf556e --- /dev/null +++ b/api-get-object.go @@ -0,0 +1,659 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// GetObject - returns an seekable, readable object. +func (c Client) GetObject(bucketName, objectName string, opts GetObjectOptions) (*Object, error) { + return c.getObjectWithContext(context.Background(), bucketName, objectName, opts) +} + +// GetObject wrapper function that accepts a request context +func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return nil, err + } + + var httpReader io.ReadCloser + var objectInfo ObjectInfo + var err error + + // Create request channel. + reqCh := make(chan getRequest) + // Create response channel. + resCh := make(chan getResponse) + // Create done channel. + doneCh := make(chan struct{}) + + // This routine feeds partial object data as and when the caller reads. + go func() { + defer close(reqCh) + defer close(resCh) + + // Used to verify if etag of object has changed since last read. + var etag string + + // Loop through the incoming control messages and read data. + for { + select { + // When the done channel is closed exit our routine. + case <-doneCh: + // Close the http response body before returning. + // This ends the connection with the server. + if httpReader != nil { + httpReader.Close() + } + return + + // Gather incoming request. + case req := <-reqCh: + // If this is the first request we may not need to do a getObject request yet. + if req.isFirstReq { + // First request is a Read/ReadAt. + if req.isReadOp { + // Differentiate between wanting the whole object and just a range. + if req.isReadAt { + // If this is a ReadAt request only get the specified range. + // Range is set with respect to the offset and length of the buffer requested. + // Do not set objectInfo from the first readAt request because it will not get + // the whole object. + opts.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1) + } else if req.Offset > 0 { + opts.SetRange(req.Offset, 0) + } + httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, opts) + if err != nil { + resCh <- getResponse{Error: err} + return + } + etag = objectInfo.ETag + // Read at least firstReq.Buffer bytes, if not we have + // reached our EOF. + size, err := io.ReadFull(httpReader, req.Buffer) + if size > 0 && err == io.ErrUnexpectedEOF { + // If an EOF happens after reading some but not + // all the bytes ReadFull returns ErrUnexpectedEOF + err = io.EOF + } + // Send back the first response. + resCh <- getResponse{ + objectInfo: objectInfo, + Size: int(size), + Error: err, + didRead: true, + } + } else { + // First request is a Stat or Seek call. + // Only need to run a StatObject until an actual Read or ReadAt request comes through. + + // Remove range header if already set, for stat Operations to get original file size. + delete(opts.headers, "Range") + objectInfo, err = c.statObject(ctx, bucketName, objectName, StatObjectOptions{opts}) + if err != nil { + resCh <- getResponse{ + Error: err, + } + // Exit the go-routine. + return + } + etag = objectInfo.ETag + // Send back the first response. + resCh <- getResponse{ + objectInfo: objectInfo, + } + } + } else if req.settingObjectInfo { // Request is just to get objectInfo. + // Remove range header if already set, for stat Operations to get original file size. + delete(opts.headers, "Range") + if etag != "" { + opts.SetMatchETag(etag) + } + objectInfo, err := c.statObject(ctx, bucketName, objectName, StatObjectOptions{opts}) + if err != nil { + resCh <- getResponse{ + Error: err, + } + // Exit the goroutine. + return + } + // Send back the objectInfo. + resCh <- getResponse{ + objectInfo: objectInfo, + } + } else { + // Offset changes fetch the new object at an Offset. + // Because the httpReader may not be set by the first + // request if it was a stat or seek it must be checked + // if the object has been read or not to only initialize + // new ones when they haven't been already. + // All readAt requests are new requests. + if req.DidOffsetChange || !req.beenRead { + if etag != "" { + opts.SetMatchETag(etag) + } + if httpReader != nil { + // Close previously opened http reader. + httpReader.Close() + } + // If this request is a readAt only get the specified range. + if req.isReadAt { + // Range is set with respect to the offset and length of the buffer requested. + opts.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1) + } else if req.Offset > 0 { // Range is set with respect to the offset. + opts.SetRange(req.Offset, 0) + } + httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, opts) + if err != nil { + resCh <- getResponse{ + Error: err, + } + return + } + } + + // Read at least req.Buffer bytes, if not we have + // reached our EOF. + size, err := io.ReadFull(httpReader, req.Buffer) + if err == io.ErrUnexpectedEOF { + // If an EOF happens after reading some but not + // all the bytes ReadFull returns ErrUnexpectedEOF + err = io.EOF + } + // Reply back how much was read. + resCh <- getResponse{ + Size: int(size), + Error: err, + didRead: true, + objectInfo: objectInfo, + } + } + } + } + }() + + // Create a newObject through the information sent back by reqCh. + return newObject(reqCh, resCh, doneCh), nil +} + +// get request message container to communicate with internal +// go-routine. +type getRequest struct { + Buffer []byte + Offset int64 // readAt offset. + DidOffsetChange bool // Tracks the offset changes for Seek requests. + beenRead bool // Determines if this is the first time an object is being read. + isReadAt bool // Determines if this request is a request to a specific range + isReadOp bool // Determines if this request is a Read or Read/At request. + isFirstReq bool // Determines if this request is the first time an object is being accessed. + settingObjectInfo bool // Determines if this request is to set the objectInfo of an object. +} + +// get response message container to reply back for the request. +type getResponse struct { + Size int + Error error + didRead bool // Lets subsequent calls know whether or not httpReader has been initiated. + objectInfo ObjectInfo // Used for the first request. +} + +// Object represents an open object. It implements +// Reader, ReaderAt, Seeker, Closer for a HTTP stream. +type Object struct { + // Mutex. + mutex *sync.Mutex + + // User allocated and defined. + reqCh chan<- getRequest + resCh <-chan getResponse + doneCh chan<- struct{} + currOffset int64 + objectInfo ObjectInfo + + // Ask lower level to initiate data fetching based on currOffset + seekData bool + + // Keeps track of closed call. + isClosed bool + + // Keeps track of if this is the first call. + isStarted bool + + // Previous error saved for future calls. + prevErr error + + // Keeps track of if this object has been read yet. + beenRead bool + + // Keeps track of if objectInfo has been set yet. + objectInfoSet bool +} + +// doGetRequest - sends and blocks on the firstReqCh and reqCh of an object. +// Returns back the size of the buffer read, if anything was read, as well +// as any error encountered. For all first requests sent on the object +// it is also responsible for sending back the objectInfo. +func (o *Object) doGetRequest(request getRequest) (getResponse, error) { + o.reqCh <- request + response := <-o.resCh + + // Return any error to the top level. + if response.Error != nil { + return response, response.Error + } + + // This was the first request. + if !o.isStarted { + // The object has been operated on. + o.isStarted = true + } + // Set the objectInfo if the request was not readAt + // and it hasn't been set before. + if !o.objectInfoSet && !request.isReadAt { + o.objectInfo = response.objectInfo + o.objectInfoSet = true + } + // Set beenRead only if it has not been set before. + if !o.beenRead { + o.beenRead = response.didRead + } + // Data are ready on the wire, no need to reinitiate connection in lower level + o.seekData = false + + return response, nil +} + +// setOffset - handles the setting of offsets for +// Read/ReadAt/Seek requests. +func (o *Object) setOffset(bytesRead int64) error { + // Update the currentOffset. + o.currOffset += bytesRead + + if o.objectInfo.Size > -1 && o.currOffset >= o.objectInfo.Size { + return io.EOF + } + return nil +} + +// Read reads up to len(b) bytes into b. It returns the number of +// bytes read (0 <= n <= len(b)) and any error encountered. Returns +// io.EOF upon end of file. +func (o *Object) Read(b []byte) (n int, err error) { + if o == nil { + return 0, ErrInvalidArgument("Object is nil") + } + + // Locking. + o.mutex.Lock() + defer o.mutex.Unlock() + + // prevErr is previous error saved from previous operation. + if o.prevErr != nil || o.isClosed { + return 0, o.prevErr + } + // Create a new request. + readReq := getRequest{ + isReadOp: true, + beenRead: o.beenRead, + Buffer: b, + } + + // Alert that this is the first request. + if !o.isStarted { + readReq.isFirstReq = true + } + + // Ask to establish a new data fetch routine based on seekData flag + readReq.DidOffsetChange = o.seekData + readReq.Offset = o.currOffset + + // Send and receive from the first request. + response, err := o.doGetRequest(readReq) + if err != nil && err != io.EOF { + // Save the error for future calls. + o.prevErr = err + return response.Size, err + } + + // Bytes read. + bytesRead := int64(response.Size) + + // Set the new offset. + oerr := o.setOffset(bytesRead) + if oerr != nil { + // Save the error for future calls. + o.prevErr = oerr + return response.Size, oerr + } + + // Return the response. + return response.Size, err +} + +// Stat returns the ObjectInfo structure describing Object. +func (o *Object) Stat() (ObjectInfo, error) { + if o == nil { + return ObjectInfo{}, ErrInvalidArgument("Object is nil") + } + // Locking. + o.mutex.Lock() + defer o.mutex.Unlock() + + if o.prevErr != nil && o.prevErr != io.EOF || o.isClosed { + return ObjectInfo{}, o.prevErr + } + + // This is the first request. + if !o.isStarted || !o.objectInfoSet { + // Send the request and get the response. + _, err := o.doGetRequest(getRequest{ + isFirstReq: !o.isStarted, + settingObjectInfo: !o.objectInfoSet, + }) + if err != nil { + o.prevErr = err + return ObjectInfo{}, err + } + } + + return o.objectInfo, nil +} + +// ReadAt reads len(b) bytes from the File starting at byte offset +// off. It returns the number of bytes read and the error, if any. +// ReadAt always returns a non-nil error when n < len(b). At end of +// file, that error is io.EOF. +func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { + if o == nil { + return 0, ErrInvalidArgument("Object is nil") + } + + // Locking. + o.mutex.Lock() + defer o.mutex.Unlock() + + // prevErr is error which was saved in previous operation. + if o.prevErr != nil || o.isClosed { + return 0, o.prevErr + } + + // Can only compare offsets to size when size has been set. + if o.objectInfoSet { + // If offset is negative than we return io.EOF. + // If offset is greater than or equal to object size we return io.EOF. + if (o.objectInfo.Size > -1 && offset >= o.objectInfo.Size) || offset < 0 { + return 0, io.EOF + } + } + + // Create the new readAt request. + readAtReq := getRequest{ + isReadOp: true, + isReadAt: true, + DidOffsetChange: true, // Offset always changes. + beenRead: o.beenRead, // Set if this is the first request to try and read. + Offset: offset, // Set the offset. + Buffer: b, + } + + // Alert that this is the first request. + if !o.isStarted { + readAtReq.isFirstReq = true + } + + // Send and receive from the first request. + response, err := o.doGetRequest(readAtReq) + if err != nil && err != io.EOF { + // Save the error. + o.prevErr = err + return response.Size, err + } + // Bytes read. + bytesRead := int64(response.Size) + // There is no valid objectInfo yet + // to compare against for EOF. + if !o.objectInfoSet { + // Update the currentOffset. + o.currOffset += bytesRead + } else { + // If this was not the first request update + // the offsets and compare against objectInfo + // for EOF. + oerr := o.setOffset(bytesRead) + if oerr != nil { + o.prevErr = oerr + return response.Size, oerr + } + } + return response.Size, err +} + +// Seek sets the offset for the next Read or Write to offset, +// interpreted according to whence: 0 means relative to the +// origin of the file, 1 means relative to the current offset, +// and 2 means relative to the end. +// Seek returns the new offset and an error, if any. +// +// Seeking to a negative offset is an error. Seeking to any positive +// offset is legal, subsequent io operations succeed until the +// underlying object is not closed. +func (o *Object) Seek(offset int64, whence int) (n int64, err error) { + if o == nil { + return 0, ErrInvalidArgument("Object is nil") + } + + // Locking. + o.mutex.Lock() + defer o.mutex.Unlock() + + if o.prevErr != nil { + // At EOF seeking is legal allow only io.EOF, for any other errors we return. + if o.prevErr != io.EOF { + return 0, o.prevErr + } + } + + // Negative offset is valid for whence of '2'. + if offset < 0 && whence != 2 { + return 0, ErrInvalidArgument(fmt.Sprintf("Negative position not allowed for %d", whence)) + } + + // This is the first request. So before anything else + // get the ObjectInfo. + if !o.isStarted || !o.objectInfoSet { + // Create the new Seek request. + seekReq := getRequest{ + isReadOp: false, + Offset: offset, + isFirstReq: true, + } + // Send and receive from the seek request. + _, err := o.doGetRequest(seekReq) + if err != nil { + // Save the error. + o.prevErr = err + return 0, err + } + } + + // Switch through whence. + switch whence { + default: + return 0, ErrInvalidArgument(fmt.Sprintf("Invalid whence %d", whence)) + case 0: + if o.objectInfo.Size > -1 && offset > o.objectInfo.Size { + return 0, io.EOF + } + o.currOffset = offset + case 1: + if o.objectInfo.Size > -1 && o.currOffset+offset > o.objectInfo.Size { + return 0, io.EOF + } + o.currOffset += offset + case 2: + // If we don't know the object size return an error for io.SeekEnd + if o.objectInfo.Size < 0 { + return 0, ErrInvalidArgument("Whence END is not supported when the object size is unknown") + } + // Seeking to positive offset is valid for whence '2', but + // since we are backing a Reader we have reached 'EOF' if + // offset is positive. + if offset > 0 { + return 0, io.EOF + } + // Seeking to negative position not allowed for whence. + if o.objectInfo.Size+offset < 0 { + return 0, ErrInvalidArgument(fmt.Sprintf("Seeking at negative offset not allowed for %d", whence)) + } + o.currOffset = o.objectInfo.Size + offset + } + // Reset the saved error since we successfully seeked, let the Read + // and ReadAt decide. + if o.prevErr == io.EOF { + o.prevErr = nil + } + + // Ask lower level to fetch again from source + o.seekData = true + + // Return the effective offset. + return o.currOffset, nil +} + +// Close - The behavior of Close after the first call returns error +// for subsequent Close() calls. +func (o *Object) Close() (err error) { + if o == nil { + return ErrInvalidArgument("Object is nil") + } + // Locking. + o.mutex.Lock() + defer o.mutex.Unlock() + + // if already closed return an error. + if o.isClosed { + return o.prevErr + } + + // Close successfully. + close(o.doneCh) + + // Save for future operations. + errMsg := "Object is already closed. Bad file descriptor." + o.prevErr = errors.New(errMsg) + // Save here that we closed done channel successfully. + o.isClosed = true + return nil +} + +// newObject instantiates a new *minio.Object* +// ObjectInfo will be set by setObjectInfo +func newObject(reqCh chan<- getRequest, resCh <-chan getResponse, doneCh chan<- struct{}) *Object { + return &Object{ + mutex: &sync.Mutex{}, + reqCh: reqCh, + resCh: resCh, + doneCh: doneCh, + } +} + +// getObject - retrieve object from Object Storage. +// +// Additionally this function also takes range arguments to download the specified +// range bytes of an object. Setting offset and length = 0 will download the full object. +// +// For more information about the HTTP Range header. +// go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. +func (c Client) getObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, error) { + // Validate input arguments. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, ObjectInfo{}, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return nil, ObjectInfo{}, err + } + + // Execute GET on objectName. + resp, err := c.executeMethod(ctx, "GET", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + customHeader: opts.Header(), + contentSHA256Hex: emptySHA256Hex, + }) + if err != nil { + return nil, ObjectInfo{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + return nil, ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) + } + } + + // Trim off the odd double quotes from ETag in the beginning and end. + md5sum := strings.TrimPrefix(resp.Header.Get("ETag"), "\"") + md5sum = strings.TrimSuffix(md5sum, "\"") + + // Parse the date. + date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) + if err != nil { + msg := "Last-Modified time format not recognized. " + reportIssue + return nil, ObjectInfo{}, ErrorResponse{ + Code: "InternalError", + Message: msg, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), + } + } + + // Get content-type. + contentType := strings.TrimSpace(resp.Header.Get("Content-Type")) + if contentType == "" { + contentType = "application/octet-stream" + } + + objectStat := ObjectInfo{ + ETag: md5sum, + Key: objectName, + Size: resp.ContentLength, + LastModified: date, + ContentType: contentType, + // Extract only the relevant header keys describing the object. + // following function filters out a list of standard set of keys + // which are not part of object metadata. + Metadata: extractObjMetadata(resp.Header), + } + + // do not close body here, caller will close + return resp.Body, objectStat, nil +} diff --git a/api-get-options.go b/api-get-options.go new file mode 100644 index 0000000..dbf062d --- /dev/null +++ b/api-get-options.go @@ -0,0 +1,128 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "fmt" + "net/http" + "time" + + "github.com/minio/minio-go/pkg/encrypt" +) + +// GetObjectOptions are used to specify additional headers or options +// during GET requests. +type GetObjectOptions struct { + headers map[string]string + ServerSideEncryption encrypt.ServerSide +} + +// StatObjectOptions are used to specify additional headers or options +// during GET info/stat requests. +type StatObjectOptions struct { + GetObjectOptions +} + +// Header returns the http.Header representation of the GET options. +func (o GetObjectOptions) Header() http.Header { + headers := make(http.Header, len(o.headers)) + for k, v := range o.headers { + headers.Set(k, v) + } + if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() == encrypt.SSEC { + o.ServerSideEncryption.Marshal(headers) + } + return headers +} + +// Set adds a key value pair to the options. The +// key-value pair will be part of the HTTP GET request +// headers. +func (o *GetObjectOptions) Set(key, value string) { + if o.headers == nil { + o.headers = make(map[string]string) + } + o.headers[http.CanonicalHeaderKey(key)] = value +} + +// SetMatchETag - set match etag. +func (o *GetObjectOptions) SetMatchETag(etag string) error { + if etag == "" { + return ErrInvalidArgument("ETag cannot be empty.") + } + o.Set("If-Match", "\""+etag+"\"") + return nil +} + +// SetMatchETagExcept - set match etag except. +func (o *GetObjectOptions) SetMatchETagExcept(etag string) error { + if etag == "" { + return ErrInvalidArgument("ETag cannot be empty.") + } + o.Set("If-None-Match", "\""+etag+"\"") + return nil +} + +// SetUnmodified - set unmodified time since. +func (o *GetObjectOptions) SetUnmodified(modTime time.Time) error { + if modTime.IsZero() { + return ErrInvalidArgument("Modified since cannot be empty.") + } + o.Set("If-Unmodified-Since", modTime.Format(http.TimeFormat)) + return nil +} + +// SetModified - set modified time since. +func (o *GetObjectOptions) SetModified(modTime time.Time) error { + if modTime.IsZero() { + return ErrInvalidArgument("Modified since cannot be empty.") + } + o.Set("If-Modified-Since", modTime.Format(http.TimeFormat)) + return nil +} + +// SetRange - set the start and end offset of the object to be read. +// See https://tools.ietf.org/html/rfc7233#section-3.1 for reference. +func (o *GetObjectOptions) SetRange(start, end int64) error { + switch { + case start == 0 && end < 0: + // Read last '-end' bytes. `bytes=-N`. + o.Set("Range", fmt.Sprintf("bytes=%d", end)) + case 0 < start && end == 0: + // Read everything starting from offset + // 'start'. `bytes=N-`. + o.Set("Range", fmt.Sprintf("bytes=%d-", start)) + case 0 <= start && start <= end: + // Read everything starting at 'start' till the + // 'end'. `bytes=N-M` + o.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + default: + // All other cases such as + // bytes=-3- + // bytes=5-3 + // bytes=-2-4 + // bytes=-3-0 + // bytes=-3--2 + // are invalid. + return ErrInvalidArgument( + fmt.Sprintf( + "Invalid range specified: start=%d end=%d", + start, end)) + } + return nil +} diff --git a/api-get-policy.go b/api-get-policy.go new file mode 100644 index 0000000..12d4c59 --- /dev/null +++ b/api-get-policy.go @@ -0,0 +1,78 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "io/ioutil" + "net/http" + "net/url" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// GetBucketPolicy - get bucket policy at a given path. +func (c Client) GetBucketPolicy(bucketName string) (string, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return "", err + } + bucketPolicy, err := c.getBucketPolicy(bucketName) + if err != nil { + errResponse := ToErrorResponse(err) + if errResponse.Code == "NoSuchBucketPolicy" { + return "", nil + } + return "", err + } + return bucketPolicy, nil +} + +// Request server for current bucket policy. +func (c Client) getBucketPolicy(bucketName string) (string, error) { + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("policy", "") + + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + + defer closeResponse(resp) + if err != nil { + return "", err + } + + if resp != nil { + if resp.StatusCode != http.StatusOK { + return "", httpRespToErrorResponse(resp, bucketName, "") + } + } + + bucketPolicyBuf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + policy := string(bucketPolicyBuf) + return policy, err +} diff --git a/api-list.go b/api-list.go new file mode 100644 index 0000000..04f7573 --- /dev/null +++ b/api-list.go @@ -0,0 +1,720 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// ListBuckets list all buckets owned by this authenticated user. +// +// This call requires explicit authentication, no anonymous requests are +// allowed for listing buckets. +// +// api := client.New(....) +// for message := range api.ListBuckets() { +// fmt.Println(message) +// } +// +func (c Client) ListBuckets() ([]BucketInfo, error) { + // Execute GET on service. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{contentSHA256Hex: emptySHA256Hex}) + defer closeResponse(resp) + if err != nil { + return nil, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, "", "") + } + } + listAllMyBucketsResult := listAllMyBucketsResult{} + err = xmlDecoder(resp.Body, &listAllMyBucketsResult) + if err != nil { + return nil, err + } + return listAllMyBucketsResult.Buckets.Bucket, nil +} + +/// Bucket Read Operations. + +// ListObjectsV2 lists all objects matching the objectPrefix from +// the specified bucket. If recursion is enabled it would list +// all subdirectories and all its contents. +// +// Your input parameters are just bucketName, objectPrefix, recursive +// and a done channel for pro-actively closing the internal go +// routine. If you enable recursive as 'true' this function will +// return back all the objects in a given bucket name and object +// prefix. +// +// api := client.New(....) +// // Create a done channel. +// doneCh := make(chan struct{}) +// defer close(doneCh) +// // Recursively list all objects in 'mytestbucket' +// recursive := true +// for message := range api.ListObjectsV2("mytestbucket", "starthere", recursive, doneCh) { +// fmt.Println(message) +// } +// +func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { + // Allocate new list objects channel. + objectStatCh := make(chan ObjectInfo, 1) + // Default listing is delimited at "/" + delimiter := "/" + if recursive { + // If recursive we do not delimit. + delimiter = "" + } + + // Return object owner information by default + fetchOwner := true + + // Validate bucket name. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + defer close(objectStatCh) + objectStatCh <- ObjectInfo{ + Err: err, + } + return objectStatCh + } + + // Validate incoming object prefix. + if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { + defer close(objectStatCh) + objectStatCh <- ObjectInfo{ + Err: err, + } + return objectStatCh + } + + // Initiate list objects goroutine here. + go func(objectStatCh chan<- ObjectInfo) { + defer close(objectStatCh) + // Save continuationToken for next request. + var continuationToken string + for { + // Get list of objects a maximum of 1000 per request. + result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, 1000, "") + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } + + // If contents are available loop through and send over channel. + for _, object := range result.Contents { + select { + // Send object content. + case objectStatCh <- object: + // If receives done from the caller, return here. + case <-doneCh: + return + } + } + + // Send all common prefixes if any. + // NOTE: prefixes are only present if the request is delimited. + for _, obj := range result.CommonPrefixes { + select { + // Send object prefixes. + case objectStatCh <- ObjectInfo{ + Key: obj.Prefix, + Size: 0, + }: + // If receives done from the caller, return here. + case <-doneCh: + return + } + } + + // If continuation token present, save it for next request. + if result.NextContinuationToken != "" { + continuationToken = result.NextContinuationToken + } + + // Listing ends result is not truncated, return right here. + if !result.IsTruncated { + return + } + } + }(objectStatCh) + return objectStatCh +} + +// listObjectsV2Query - (List Objects V2) - List some or all (up to 1000) of the objects in a bucket. +// +// You can use the request parameters as selection criteria to return a subset of the objects in a bucket. +// request parameters :- +// --------- +// ?continuation-token - Used to continue iterating over a set of objects +// ?delimiter - A delimiter is a character you use to group keys. +// ?prefix - Limits the response to keys that begin with the specified prefix. +// ?max-keys - Sets the maximum number of keys returned in the response body. +// ?start-after - Specifies the key to start after when listing objects in a bucket. +func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { + // Validate bucket name. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return ListBucketV2Result{}, err + } + // Validate object prefix. + if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { + return ListBucketV2Result{}, err + } + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + + // Always set list-type in ListObjects V2 + urlValues.Set("list-type", "2") + + // Set object prefix. + if objectPrefix != "" { + urlValues.Set("prefix", objectPrefix) + } + // Set continuation token + if continuationToken != "" { + urlValues.Set("continuation-token", continuationToken) + } + // Set delimiter. + if delimiter != "" { + urlValues.Set("delimiter", delimiter) + } + + // Fetch owner when listing + if fetchOwner { + urlValues.Set("fetch-owner", "true") + } + + // maxkeys should default to 1000 or less. + if maxkeys == 0 || maxkeys > 1000 { + maxkeys = 1000 + } + // Set max keys. + urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + + // Set start-after + if startAfter != "" { + urlValues.Set("start-after", startAfter) + } + + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return ListBucketV2Result{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return ListBucketV2Result{}, httpRespToErrorResponse(resp, bucketName, "") + } + } + + // Decode listBuckets XML. + listBucketResult := ListBucketV2Result{} + if err = xmlDecoder(resp.Body, &listBucketResult); err != nil { + return listBucketResult, err + } + + // This is an additional verification check to make + // sure proper responses are received. + if listBucketResult.IsTruncated && listBucketResult.NextContinuationToken == "" { + return listBucketResult, errors.New("Truncated response should have continuation token set") + } + + // Success. + return listBucketResult, nil +} + +// ListObjects - (List Objects) - List some objects or all recursively. +// +// ListObjects lists all objects matching the objectPrefix from +// the specified bucket. If recursion is enabled it would list +// all subdirectories and all its contents. +// +// Your input parameters are just bucketName, objectPrefix, recursive +// and a done channel for pro-actively closing the internal go +// routine. If you enable recursive as 'true' this function will +// return back all the objects in a given bucket name and object +// prefix. +// +// api := client.New(....) +// // Create a done channel. +// doneCh := make(chan struct{}) +// defer close(doneCh) +// // Recurively list all objects in 'mytestbucket' +// recursive := true +// for message := range api.ListObjects("mytestbucket", "starthere", recursive, doneCh) { +// fmt.Println(message) +// } +// +func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { + // Allocate new list objects channel. + objectStatCh := make(chan ObjectInfo, 1) + // Default listing is delimited at "/" + delimiter := "/" + if recursive { + // If recursive we do not delimit. + delimiter = "" + } + // Validate bucket name. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + defer close(objectStatCh) + objectStatCh <- ObjectInfo{ + Err: err, + } + return objectStatCh + } + // Validate incoming object prefix. + if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { + defer close(objectStatCh) + objectStatCh <- ObjectInfo{ + Err: err, + } + return objectStatCh + } + + // Initiate list objects goroutine here. + go func(objectStatCh chan<- ObjectInfo) { + defer close(objectStatCh) + // Save marker for next request. + var marker string + for { + // Get list of objects a maximum of 1000 per request. + result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 1000) + if err != nil { + objectStatCh <- ObjectInfo{ + Err: err, + } + return + } + + // If contents are available loop through and send over channel. + for _, object := range result.Contents { + // Save the marker. + marker = object.Key + select { + // Send object content. + case objectStatCh <- object: + // If receives done from the caller, return here. + case <-doneCh: + return + } + } + + // Send all common prefixes if any. + // NOTE: prefixes are only present if the request is delimited. + for _, obj := range result.CommonPrefixes { + object := ObjectInfo{} + object.Key = obj.Prefix + object.Size = 0 + select { + // Send object prefixes. + case objectStatCh <- object: + // If receives done from the caller, return here. + case <-doneCh: + return + } + } + + // If next marker present, save it for next request. + if result.NextMarker != "" { + marker = result.NextMarker + } + + // Listing ends result is not truncated, return right here. + if !result.IsTruncated { + return + } + } + }(objectStatCh) + return objectStatCh +} + +// listObjects - (List Objects) - List some or all (up to 1000) of the objects in a bucket. +// +// You can use the request parameters as selection criteria to return a subset of the objects in a bucket. +// request parameters :- +// --------- +// ?marker - Specifies the key to start with when listing objects in a bucket. +// ?delimiter - A delimiter is a character you use to group keys. +// ?prefix - Limits the response to keys that begin with the specified prefix. +// ?max-keys - Sets the maximum number of keys returned in the response body. +func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (ListBucketResult, error) { + // Validate bucket name. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return ListBucketResult{}, err + } + // Validate object prefix. + if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { + return ListBucketResult{}, err + } + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + // Set object prefix. + if objectPrefix != "" { + urlValues.Set("prefix", objectPrefix) + } + // Set object marker. + if objectMarker != "" { + urlValues.Set("marker", objectMarker) + } + // Set delimiter. + if delimiter != "" { + urlValues.Set("delimiter", delimiter) + } + + // maxkeys should default to 1000 or less. + if maxkeys == 0 || maxkeys > 1000 { + maxkeys = 1000 + } + // Set max keys. + urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return ListBucketResult{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return ListBucketResult{}, httpRespToErrorResponse(resp, bucketName, "") + } + } + // Decode listBuckets XML. + listBucketResult := ListBucketResult{} + err = xmlDecoder(resp.Body, &listBucketResult) + if err != nil { + return listBucketResult, err + } + return listBucketResult, nil +} + +// ListIncompleteUploads - List incompletely uploaded multipart objects. +// +// ListIncompleteUploads lists all incompleted objects matching the +// objectPrefix from the specified bucket. If recursion is enabled +// it would list all subdirectories and all its contents. +// +// Your input parameters are just bucketName, objectPrefix, recursive +// and a done channel to pro-actively close the internal go routine. +// If you enable recursive as 'true' this function will return back all +// the multipart objects in a given bucket name. +// +// api := client.New(....) +// // Create a done channel. +// doneCh := make(chan struct{}) +// defer close(doneCh) +// // Recurively list all objects in 'mytestbucket' +// recursive := true +// for message := range api.ListIncompleteUploads("mytestbucket", "starthere", recursive) { +// fmt.Println(message) +// } +// +func (c Client) ListIncompleteUploads(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { + // Turn on size aggregation of individual parts. + isAggregateSize := true + return c.listIncompleteUploads(bucketName, objectPrefix, recursive, isAggregateSize, doneCh) +} + +// listIncompleteUploads lists all incomplete uploads. +func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive, aggregateSize bool, doneCh <-chan struct{}) <-chan ObjectMultipartInfo { + // Allocate channel for multipart uploads. + objectMultipartStatCh := make(chan ObjectMultipartInfo, 1) + // Delimiter is set to "/" by default. + delimiter := "/" + if recursive { + // If recursive do not delimit. + delimiter = "" + } + // Validate bucket name. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + defer close(objectMultipartStatCh) + objectMultipartStatCh <- ObjectMultipartInfo{ + Err: err, + } + return objectMultipartStatCh + } + // Validate incoming object prefix. + if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { + defer close(objectMultipartStatCh) + objectMultipartStatCh <- ObjectMultipartInfo{ + Err: err, + } + return objectMultipartStatCh + } + go func(objectMultipartStatCh chan<- ObjectMultipartInfo) { + defer close(objectMultipartStatCh) + // object and upload ID marker for future requests. + var objectMarker string + var uploadIDMarker string + for { + // list all multipart uploads. + result, err := c.listMultipartUploadsQuery(bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 1000) + if err != nil { + objectMultipartStatCh <- ObjectMultipartInfo{ + Err: err, + } + return + } + // Save objectMarker and uploadIDMarker for next request. + objectMarker = result.NextKeyMarker + uploadIDMarker = result.NextUploadIDMarker + // Send all multipart uploads. + for _, obj := range result.Uploads { + // Calculate total size of the uploaded parts if 'aggregateSize' is enabled. + if aggregateSize { + // Get total multipart size. + obj.Size, err = c.getTotalMultipartSize(bucketName, obj.Key, obj.UploadID) + if err != nil { + objectMultipartStatCh <- ObjectMultipartInfo{ + Err: err, + } + continue + } + } + select { + // Send individual uploads here. + case objectMultipartStatCh <- obj: + // If done channel return here. + case <-doneCh: + return + } + } + // Send all common prefixes if any. + // NOTE: prefixes are only present if the request is delimited. + for _, obj := range result.CommonPrefixes { + object := ObjectMultipartInfo{} + object.Key = obj.Prefix + object.Size = 0 + select { + // Send delimited prefixes here. + case objectMultipartStatCh <- object: + // If done channel return here. + case <-doneCh: + return + } + } + // Listing ends if result not truncated, return right here. + if !result.IsTruncated { + return + } + } + }(objectMultipartStatCh) + // return. + return objectMultipartStatCh +} + +// listMultipartUploads - (List Multipart Uploads). +// - Lists some or all (up to 1000) in-progress multipart uploads in a bucket. +// +// You can use the request parameters as selection criteria to return a subset of the uploads in a bucket. +// request parameters. :- +// --------- +// ?key-marker - Specifies the multipart upload after which listing should begin. +// ?upload-id-marker - Together with key-marker specifies the multipart upload after which listing should begin. +// ?delimiter - A delimiter is a character you use to group keys. +// ?prefix - Limits the response to keys that begin with the specified prefix. +// ?max-uploads - Sets the maximum number of multipart uploads returned in the response body. +func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, prefix, delimiter string, maxUploads int) (ListMultipartUploadsResult, error) { + // Get resources properly escaped and lined up before using them in http request. + urlValues := make(url.Values) + // Set uploads. + urlValues.Set("uploads", "") + // Set object key marker. + if keyMarker != "" { + urlValues.Set("key-marker", keyMarker) + } + // Set upload id marker. + if uploadIDMarker != "" { + urlValues.Set("upload-id-marker", uploadIDMarker) + } + // Set prefix marker. + if prefix != "" { + urlValues.Set("prefix", prefix) + } + // Set delimiter. + if delimiter != "" { + urlValues.Set("delimiter", delimiter) + } + + // maxUploads should be 1000 or less. + if maxUploads == 0 || maxUploads > 1000 { + maxUploads = 1000 + } + // Set max-uploads. + urlValues.Set("max-uploads", fmt.Sprintf("%d", maxUploads)) + + // Execute GET on bucketName to list multipart uploads. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return ListMultipartUploadsResult{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return ListMultipartUploadsResult{}, httpRespToErrorResponse(resp, bucketName, "") + } + } + // Decode response body. + listMultipartUploadsResult := ListMultipartUploadsResult{} + err = xmlDecoder(resp.Body, &listMultipartUploadsResult) + if err != nil { + return listMultipartUploadsResult, err + } + return listMultipartUploadsResult, nil +} + +// listObjectParts list all object parts recursively. +func (c Client) listObjectParts(bucketName, objectName, uploadID string) (partsInfo map[int]ObjectPart, err error) { + // Part number marker for the next batch of request. + var nextPartNumberMarker int + partsInfo = make(map[int]ObjectPart) + for { + // Get list of uploaded parts a maximum of 1000 per request. + listObjPartsResult, err := c.listObjectPartsQuery(bucketName, objectName, uploadID, nextPartNumberMarker, 1000) + if err != nil { + return nil, err + } + // Append to parts info. + for _, part := range listObjPartsResult.ObjectParts { + // Trim off the odd double quotes from ETag in the beginning and end. + part.ETag = strings.TrimPrefix(part.ETag, "\"") + part.ETag = strings.TrimSuffix(part.ETag, "\"") + partsInfo[part.PartNumber] = part + } + // Keep part number marker, for the next iteration. + nextPartNumberMarker = listObjPartsResult.NextPartNumberMarker + // Listing ends result is not truncated, return right here. + if !listObjPartsResult.IsTruncated { + break + } + } + + // Return all the parts. + return partsInfo, nil +} + +// findUploadIDs lists all incomplete uploads and find the uploadIDs of the matching object name. +func (c Client) findUploadIDs(bucketName, objectName string) ([]string, error) { + var uploadIDs []string + // Make list incomplete uploads recursive. + isRecursive := true + // Turn off size aggregation of individual parts, in this request. + isAggregateSize := false + // Create done channel to cleanup the routine. + doneCh := make(chan struct{}) + defer close(doneCh) + // List all incomplete uploads. + for mpUpload := range c.listIncompleteUploads(bucketName, objectName, isRecursive, isAggregateSize, doneCh) { + if mpUpload.Err != nil { + return nil, mpUpload.Err + } + if objectName == mpUpload.Key { + uploadIDs = append(uploadIDs, mpUpload.UploadID) + } + } + // Return the latest upload id. + return uploadIDs, nil +} + +// getTotalMultipartSize - calculate total uploaded size for the a given multipart object. +func (c Client) getTotalMultipartSize(bucketName, objectName, uploadID string) (size int64, err error) { + // Iterate over all parts and aggregate the size. + partsInfo, err := c.listObjectParts(bucketName, objectName, uploadID) + if err != nil { + return 0, err + } + for _, partInfo := range partsInfo { + size += partInfo.Size + } + return size, nil +} + +// listObjectPartsQuery (List Parts query) +// - lists some or all (up to 1000) parts that have been uploaded +// for a specific multipart upload +// +// You can use the request parameters as selection criteria to return +// a subset of the uploads in a bucket, request parameters :- +// --------- +// ?part-number-marker - Specifies the part after which listing should +// begin. +// ?max-parts - Maximum parts to be listed per request. +func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, partNumberMarker, maxParts int) (ListObjectPartsResult, error) { + // Get resources properly escaped and lined up before using them in http request. + urlValues := make(url.Values) + // Set part number marker. + urlValues.Set("part-number-marker", fmt.Sprintf("%d", partNumberMarker)) + // Set upload id. + urlValues.Set("uploadId", uploadID) + + // maxParts should be 1000 or less. + if maxParts == 0 || maxParts > 1000 { + maxParts = 1000 + } + // Set max parts. + urlValues.Set("max-parts", fmt.Sprintf("%d", maxParts)) + + // Execute GET on objectName to get list of parts. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return ListObjectPartsResult{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return ListObjectPartsResult{}, httpRespToErrorResponse(resp, bucketName, objectName) + } + } + // Decode list object parts XML. + listObjectPartsResult := ListObjectPartsResult{} + err = xmlDecoder(resp.Body, &listObjectPartsResult) + if err != nil { + return listObjectPartsResult, err + } + return listObjectPartsResult, nil +} diff --git a/api-notification.go b/api-notification.go new file mode 100644 index 0000000..1c01e36 --- /dev/null +++ b/api-notification.go @@ -0,0 +1,228 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bufio" + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// GetBucketNotification - get bucket notification at a given path. +func (c Client) GetBucketNotification(bucketName string) (bucketNotification BucketNotification, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return BucketNotification{}, err + } + notification, err := c.getBucketNotification(bucketName) + if err != nil { + return BucketNotification{}, err + } + return notification, nil +} + +// Request server for notification rules. +func (c Client) getBucketNotification(bucketName string) (BucketNotification, error) { + urlValues := make(url.Values) + urlValues.Set("notification", "") + + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + + defer closeResponse(resp) + if err != nil { + return BucketNotification{}, err + } + return processBucketNotificationResponse(bucketName, resp) + +} + +// processes the GetNotification http response from the server. +func processBucketNotificationResponse(bucketName string, resp *http.Response) (BucketNotification, error) { + if resp.StatusCode != http.StatusOK { + errResponse := httpRespToErrorResponse(resp, bucketName, "") + return BucketNotification{}, errResponse + } + var bucketNotification BucketNotification + err := xmlDecoder(resp.Body, &bucketNotification) + if err != nil { + return BucketNotification{}, err + } + return bucketNotification, nil +} + +// Indentity represents the user id, this is a compliance field. +type identity struct { + PrincipalID string `json:"principalId"` +} + +// Notification event bucket metadata. +type bucketMeta struct { + Name string `json:"name"` + OwnerIdentity identity `json:"ownerIdentity"` + ARN string `json:"arn"` +} + +// Notification event object metadata. +type objectMeta struct { + Key string `json:"key"` + Size int64 `json:"size,omitempty"` + ETag string `json:"eTag,omitempty"` + VersionID string `json:"versionId,omitempty"` + Sequencer string `json:"sequencer"` +} + +// Notification event server specific metadata. +type eventMeta struct { + SchemaVersion string `json:"s3SchemaVersion"` + ConfigurationID string `json:"configurationId"` + Bucket bucketMeta `json:"bucket"` + Object objectMeta `json:"object"` +} + +// sourceInfo represents information on the client that +// triggered the event notification. +type sourceInfo struct { + Host string `json:"host"` + Port string `json:"port"` + UserAgent string `json:"userAgent"` +} + +// NotificationEvent represents an Amazon an S3 bucket notification event. +type NotificationEvent struct { + EventVersion string `json:"eventVersion"` + EventSource string `json:"eventSource"` + AwsRegion string `json:"awsRegion"` + EventTime string `json:"eventTime"` + EventName string `json:"eventName"` + UserIdentity identity `json:"userIdentity"` + RequestParameters map[string]string `json:"requestParameters"` + ResponseElements map[string]string `json:"responseElements"` + S3 eventMeta `json:"s3"` + Source sourceInfo `json:"source"` +} + +// NotificationInfo - represents the collection of notification events, additionally +// also reports errors if any while listening on bucket notifications. +type NotificationInfo struct { + Records []NotificationEvent + Err error +} + +// ListenBucketNotification - listen on bucket notifications. +func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo { + notificationInfoCh := make(chan NotificationInfo, 1) + // Only success, start a routine to start reading line by line. + go func(notificationInfoCh chan<- NotificationInfo) { + defer close(notificationInfoCh) + + // Validate the bucket name. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + notificationInfoCh <- NotificationInfo{ + Err: err, + } + return + } + + // Check ARN partition to verify if listening bucket is supported + if s3utils.IsAmazonEndpoint(*c.endpointURL) || s3utils.IsGoogleEndpoint(*c.endpointURL) { + notificationInfoCh <- NotificationInfo{ + Err: ErrAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"), + } + return + } + + // Continuously run and listen on bucket notification. + // Create a done channel to control 'ListObjects' go routine. + retryDoneCh := make(chan struct{}, 1) + + // Indicate to our routine to exit cleanly upon return. + defer close(retryDoneCh) + + // Wait on the jitter retry loop. + for range c.newRetryTimerContinous(time.Second, time.Second*30, MaxJitter, retryDoneCh) { + urlValues := make(url.Values) + urlValues.Set("prefix", prefix) + urlValues.Set("suffix", suffix) + urlValues["events"] = events + + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + if err != nil { + notificationInfoCh <- NotificationInfo{ + Err: err, + } + return + } + + // Validate http response, upon error return quickly. + if resp.StatusCode != http.StatusOK { + errResponse := httpRespToErrorResponse(resp, bucketName, "") + notificationInfoCh <- NotificationInfo{ + Err: errResponse, + } + return + } + + // Initialize a new bufio scanner, to read line by line. + bio := bufio.NewScanner(resp.Body) + + // Close the response body. + defer resp.Body.Close() + + // Unmarshal each line, returns marshalled values. + for bio.Scan() { + var notificationInfo NotificationInfo + if err = json.Unmarshal(bio.Bytes(), ¬ificationInfo); err != nil { + continue + } + // Send notificationInfo + select { + case notificationInfoCh <- notificationInfo: + case <-doneCh: + return + } + } + // Look for any underlying errors. + if err = bio.Err(); err != nil { + // For an unexpected connection drop from server, we close the body + // and re-connect. + if err == io.ErrUnexpectedEOF { + resp.Body.Close() + } + } + } + }(notificationInfoCh) + + // Returns the notification info channel, for caller to start reading from. + return notificationInfoCh +} diff --git a/api-presigned.go b/api-presigned.go new file mode 100644 index 0000000..a2c0607 --- /dev/null +++ b/api-presigned.go @@ -0,0 +1,215 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "errors" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/pkg/s3signer" + "github.com/minio/minio-go/pkg/s3utils" +) + +// presignURL - Returns a presigned URL for an input 'method'. +// Expires maximum is 7days - ie. 604800 and minimum is 1. +func (c Client) presignURL(method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { + // Input validation. + if method == "" { + return nil, ErrInvalidArgument("method cannot be empty.") + } + if err = s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, err + } + if err = isValidExpiry(expires); err != nil { + return nil, err + } + + // Convert expires into seconds. + expireSeconds := int64(expires / time.Second) + reqMetadata := requestMetadata{ + presignURL: true, + bucketName: bucketName, + objectName: objectName, + expires: expireSeconds, + queryValues: reqParams, + } + + // Instantiate a new request. + // Since expires is set newRequest will presign the request. + var req *http.Request + if req, err = c.newRequest(method, reqMetadata); err != nil { + return nil, err + } + return req.URL, nil +} + +// PresignedGetObject - Returns a presigned URL to access an object +// data without credentials. URL can have a maximum expiry of +// upto 7days or a minimum of 1sec. Additionally you can override +// a set of response headers using the query parameters. +func (c Client) PresignedGetObject(bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { + if err = s3utils.CheckValidObjectName(objectName); err != nil { + return nil, err + } + return c.presignURL("GET", bucketName, objectName, expires, reqParams) +} + +// PresignedHeadObject - Returns a presigned URL to access object +// metadata without credentials. URL can have a maximum expiry of +// upto 7days or a minimum of 1sec. Additionally you can override +// a set of response headers using the query parameters. +func (c Client) PresignedHeadObject(bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { + if err = s3utils.CheckValidObjectName(objectName); err != nil { + return nil, err + } + return c.presignURL("HEAD", bucketName, objectName, expires, reqParams) +} + +// PresignedPutObject - Returns a presigned URL to upload an object +// without credentials. URL can have a maximum expiry of upto 7days +// or a minimum of 1sec. +func (c Client) PresignedPutObject(bucketName string, objectName string, expires time.Duration) (u *url.URL, err error) { + if err = s3utils.CheckValidObjectName(objectName); err != nil { + return nil, err + } + return c.presignURL("PUT", bucketName, objectName, expires, nil) +} + +// Presign - returns a presigned URL for any http method of your choice +// along with custom request params. URL can have a maximum expiry of +// upto 7days or a minimum of 1sec. +func (c Client) Presign(method string, bucketName string, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { + return c.presignURL(method, bucketName, objectName, expires, reqParams) +} + +// PresignedPostPolicy - Returns POST urlString, form data to upload an object. +func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[string]string, err error) { + // Validate input arguments. + if p.expiration.IsZero() { + return nil, nil, errors.New("Expiration time must be specified") + } + if _, ok := p.formData["key"]; !ok { + return nil, nil, errors.New("object key must be specified") + } + if _, ok := p.formData["bucket"]; !ok { + return nil, nil, errors.New("bucket name must be specified") + } + + bucketName := p.formData["bucket"] + // Fetch the bucket location. + location, err := c.getBucketLocation(bucketName) + if err != nil { + return nil, nil, err + } + + isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, bucketName) + + u, err = c.makeTargetURL(bucketName, "", location, isVirtualHost, nil) + if err != nil { + return nil, nil, err + } + + // Get credentials from the configured credentials provider. + credValues, err := c.credsProvider.Get() + if err != nil { + return nil, nil, err + } + + var ( + signerType = credValues.SignerType + sessionToken = credValues.SessionToken + accessKeyID = credValues.AccessKeyID + secretAccessKey = credValues.SecretAccessKey + ) + + if signerType.IsAnonymous() { + return nil, nil, ErrInvalidArgument("Presigned operations are not supported for anonymous credentials") + } + + // Keep time. + t := time.Now().UTC() + // For signature version '2' handle here. + if signerType.IsV2() { + policyBase64 := p.base64() + p.formData["policy"] = policyBase64 + // For Google endpoint set this value to be 'GoogleAccessId'. + if s3utils.IsGoogleEndpoint(*c.endpointURL) { + p.formData["GoogleAccessId"] = accessKeyID + } else { + // For all other endpoints set this value to be 'AWSAccessKeyId'. + p.formData["AWSAccessKeyId"] = accessKeyID + } + // Sign the policy. + p.formData["signature"] = s3signer.PostPresignSignatureV2(policyBase64, secretAccessKey) + return u, p.formData, nil + } + + // Add date policy. + if err = p.addNewPolicy(policyCondition{ + matchType: "eq", + condition: "$x-amz-date", + value: t.Format(iso8601DateFormat), + }); err != nil { + return nil, nil, err + } + + // Add algorithm policy. + if err = p.addNewPolicy(policyCondition{ + matchType: "eq", + condition: "$x-amz-algorithm", + value: signV4Algorithm, + }); err != nil { + return nil, nil, err + } + + // Add a credential policy. + credential := s3signer.GetCredential(accessKeyID, location, t) + if err = p.addNewPolicy(policyCondition{ + matchType: "eq", + condition: "$x-amz-credential", + value: credential, + }); err != nil { + return nil, nil, err + } + + if sessionToken != "" { + if err = p.addNewPolicy(policyCondition{ + matchType: "eq", + condition: "$x-amz-security-token", + value: sessionToken, + }); err != nil { + return nil, nil, err + } + } + + // Get base64 encoded policy. + policyBase64 := p.base64() + + // Fill in the form data. + p.formData["policy"] = policyBase64 + p.formData["x-amz-algorithm"] = signV4Algorithm + p.formData["x-amz-credential"] = credential + p.formData["x-amz-date"] = t.Format(iso8601DateFormat) + if sessionToken != "" { + p.formData["x-amz-security-token"] = sessionToken + } + p.formData["x-amz-signature"] = s3signer.PostPresignSignatureV4(policyBase64, t, secretAccessKey, location) + return u, p.formData, nil +} diff --git a/api-put-bucket.go b/api-put-bucket.go new file mode 100644 index 0000000..33dc0cf --- /dev/null +++ b/api-put-bucket.go @@ -0,0 +1,306 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/minio/minio-go/pkg/s3utils" +) + +/// Bucket operations + +// MakeBucket creates a new bucket with bucketName. +// +// Location is an optional argument, by default all buckets are +// created in US Standard Region. +// +// For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html +// For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations +func (c Client) MakeBucket(bucketName string, location string) (err error) { + defer func() { + // Save the location into cache on a successful makeBucket response. + if err == nil { + c.bucketLocCache.Set(bucketName, location) + } + }() + + // Validate the input arguments. + if err := s3utils.CheckValidBucketNameStrict(bucketName); err != nil { + return err + } + + // If location is empty, treat is a default region 'us-east-1'. + if location == "" { + location = "us-east-1" + // For custom region clients, default + // to custom region instead not 'us-east-1'. + if c.region != "" { + location = c.region + } + } + // PUT bucket request metadata. + reqMetadata := requestMetadata{ + bucketName: bucketName, + bucketLocation: location, + } + + // If location is not 'us-east-1' create bucket location config. + if location != "us-east-1" && location != "" { + createBucketConfig := createBucketConfiguration{} + createBucketConfig.Location = location + var createBucketConfigBytes []byte + createBucketConfigBytes, err = xml.Marshal(createBucketConfig) + if err != nil { + return err + } + reqMetadata.contentMD5Base64 = sumMD5Base64(createBucketConfigBytes) + reqMetadata.contentSHA256Hex = sum256Hex(createBucketConfigBytes) + reqMetadata.contentBody = bytes.NewReader(createBucketConfigBytes) + reqMetadata.contentLength = int64(len(createBucketConfigBytes)) + } + + // Execute PUT to create a new bucket. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + + // Success. + return nil +} + +// SetBucketPolicy set the access permissions on an existing bucket. +func (c Client) SetBucketPolicy(bucketName, policy string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // If policy is empty then delete the bucket policy. + if policy == "" { + return c.removeBucketPolicy(bucketName) + } + + // Save the updated policies. + return c.putBucketPolicy(bucketName, policy) +} + +// Saves a new bucket policy. +func (c Client) putBucketPolicy(bucketName, policy string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("policy", "") + + // Content-length is mandatory for put policy request + policyReader := strings.NewReader(policy) + b, err := ioutil.ReadAll(policyReader) + if err != nil { + return err + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: policyReader, + contentLength: int64(len(b)), + } + + // Execute PUT to upload a new bucket policy. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// Removes all policies on a bucket. +func (c Client) removeBucketPolicy(bucketName string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("policy", "") + + // Execute DELETE on objectName. + resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + return nil +} + +// SetBucketLifecycle set the lifecycle on an existing bucket. +func (c Client) SetBucketLifecycle(bucketName, lifecycle string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // If lifecycle is empty then delete it. + if lifecycle == "" { + return c.removeBucketLifecycle(bucketName) + } + + // Save the updated lifecycle. + return c.putBucketLifecycle(bucketName, lifecycle) +} + +// Saves a new bucket lifecycle. +func (c Client) putBucketLifecycle(bucketName, lifecycle string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("lifecycle", "") + + // Content-length is mandatory for put lifecycle request + lifecycleReader := strings.NewReader(lifecycle) + b, err := ioutil.ReadAll(lifecycleReader) + if err != nil { + return err + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: lifecycleReader, + contentLength: int64(len(b)), + contentMD5Base64: sumMD5Base64(b), + } + + // Execute PUT to upload a new bucket lifecycle. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// Remove lifecycle from a bucket. +func (c Client) removeBucketLifecycle(bucketName string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("lifecycle", "") + + // Execute DELETE on objectName. + resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + return nil +} + +// SetBucketNotification saves a new bucket notification. +func (c Client) SetBucketNotification(bucketName string, bucketNotification BucketNotification) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("notification", "") + + notifBytes, err := xml.Marshal(bucketNotification) + if err != nil { + return err + } + + notifBuffer := bytes.NewReader(notifBytes) + reqMetadata := requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: notifBuffer, + contentLength: int64(len(notifBytes)), + contentMD5Base64: sumMD5Base64(notifBytes), + contentSHA256Hex: sum256Hex(notifBytes), + } + + // Execute PUT to upload a new bucket notification. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + return nil +} + +// RemoveAllBucketNotification - Remove bucket notification clears all previously specified config +func (c Client) RemoveAllBucketNotification(bucketName string) error { + return c.SetBucketNotification(bucketName, BucketNotification{}) +} diff --git a/api-put-object-common.go b/api-put-object-common.go new file mode 100644 index 0000000..c16c3c6 --- /dev/null +++ b/api-put-object-common.go @@ -0,0 +1,111 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "io" + "math" + "os" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// Verify if reader is *minio.Object +func isObject(reader io.Reader) (ok bool) { + _, ok = reader.(*Object) + return +} + +// Verify if reader is a generic ReaderAt +func isReadAt(reader io.Reader) (ok bool) { + _, ok = reader.(io.ReaderAt) + if ok { + var v *os.File + v, ok = reader.(*os.File) + if ok { + // Stdin, Stdout and Stderr all have *os.File type + // which happen to also be io.ReaderAt compatible + // we need to add special conditions for them to + // be ignored by this function. + for _, f := range []string{ + "/dev/stdin", + "/dev/stdout", + "/dev/stderr", + } { + if f == v.Name() { + ok = false + break + } + } + } + } + return +} + +// optimalPartInfo - calculate the optimal part info for a given +// object size. +// +// NOTE: Assumption here is that for any object to be uploaded to any S3 compatible +// object storage it will have the following parameters as constants. +// +// maxPartsCount - 10000 +// minPartSize - 64MiB +// maxMultipartPutObjectSize - 5TiB +// +func optimalPartInfo(objectSize int64) (totalPartsCount int, partSize int64, lastPartSize int64, err error) { + // object size is '-1' set it to 5TiB. + if objectSize == -1 { + objectSize = maxMultipartPutObjectSize + } + // object size is larger than supported maximum. + if objectSize > maxMultipartPutObjectSize { + err = ErrEntityTooLarge(objectSize, maxMultipartPutObjectSize, "", "") + return + } + // Use floats for part size for all calculations to avoid + // overflows during float64 to int64 conversions. + partSizeFlt := math.Ceil(float64(objectSize / maxPartsCount)) + partSizeFlt = math.Ceil(partSizeFlt/minPartSize) * minPartSize + // Total parts count. + totalPartsCount = int(math.Ceil(float64(objectSize) / partSizeFlt)) + // Part size. + partSize = int64(partSizeFlt) + // Last part size. + lastPartSize = objectSize - int64(totalPartsCount-1)*partSize + return totalPartsCount, partSize, lastPartSize, nil +} + +// getUploadID - fetch upload id if already present for an object name +// or initiate a new request to fetch a new upload id. +func (c Client) newUploadID(ctx context.Context, bucketName, objectName string, opts PutObjectOptions) (uploadID string, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return "", err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return "", err + } + + // Initiate multipart upload for an object. + initMultipartUploadResult, err := c.initiateMultipartUpload(ctx, bucketName, objectName, opts) + if err != nil { + return "", err + } + return initMultipartUploadResult.UploadID, nil +} diff --git a/api-put-object-context.go b/api-put-object-context.go new file mode 100644 index 0000000..ff4663e --- /dev/null +++ b/api-put-object-context.go @@ -0,0 +1,33 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "io" +) + +// PutObjectWithContext - Identical to PutObject call, but accepts context to facilitate request cancellation. +func (c Client) PutObjectWithContext(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, + opts PutObjectOptions) (n int64, err error) { + err = opts.validate() + if err != nil { + return 0, err + } + return c.putObjectCommon(ctx, bucketName, objectName, reader, objectSize, opts) +} diff --git a/api-put-object-copy.go b/api-put-object-copy.go new file mode 100644 index 0000000..21322ef --- /dev/null +++ b/api-put-object-copy.go @@ -0,0 +1,83 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017, 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "io" + "io/ioutil" + "net/http" + + "github.com/minio/minio-go/pkg/encrypt" +) + +// CopyObject - copy a source object into a new object +func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error { + return c.CopyObjectWithProgress(dst, src, nil) +} + +// CopyObjectWithProgress - copy a source object into a new object, optionally takes +// progress bar input to notify current progress. +func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, progress io.Reader) error { + header := make(http.Header) + for k, v := range src.Headers { + header[k] = v + } + + var err error + var size int64 + // If progress bar is specified, size should be requested as well initiate a StatObject request. + if progress != nil { + size, _, _, err = src.getProps(c) + if err != nil { + return err + } + } + + if src.encryption != nil { + encrypt.SSECopy(src.encryption).Marshal(header) + } + + if dst.encryption != nil { + dst.encryption.Marshal(header) + } + for k, v := range dst.getUserMetaHeadersMap(true) { + header.Set(k, v) + } + + resp, err := c.executeMethod(context.Background(), "PUT", requestMetadata{ + bucketName: dst.bucket, + objectName: dst.object, + customHeader: header, + }) + if err != nil { + return err + } + defer closeResponse(resp) + + if resp.StatusCode != http.StatusOK { + return httpRespToErrorResponse(resp, dst.bucket, dst.object) + } + + // Update the progress properly after successful copy. + if progress != nil { + io.CopyN(ioutil.Discard, progress, size) + } + + return nil +} diff --git a/api-put-object-file-context.go b/api-put-object-file-context.go new file mode 100644 index 0000000..140a9c0 --- /dev/null +++ b/api-put-object-file-context.go @@ -0,0 +1,64 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "mime" + "os" + "path/filepath" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// FPutObjectWithContext - Create an object in a bucket, with contents from file at filePath. Allows request cancellation. +func (c Client) FPutObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts PutObjectOptions) (n int64, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return 0, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return 0, err + } + + // Open the referenced file. + fileReader, err := os.Open(filePath) + // If any error fail quickly here. + if err != nil { + return 0, err + } + defer fileReader.Close() + + // Save the file stat. + fileStat, err := fileReader.Stat() + if err != nil { + return 0, err + } + + // Save the file size. + fileSize := fileStat.Size() + + // Set contentType based on filepath extension if not given or default + // value of "application/octet-stream" if the extension has no associated type. + if opts.ContentType == "" { + if opts.ContentType = mime.TypeByExtension(filepath.Ext(filePath)); opts.ContentType == "" { + opts.ContentType = "application/octet-stream" + } + } + return c.PutObjectWithContext(ctx, bucketName, objectName, fileReader, fileSize, opts) +} diff --git a/api-put-object-file.go b/api-put-object-file.go new file mode 100644 index 0000000..7c8e051 --- /dev/null +++ b/api-put-object-file.go @@ -0,0 +1,27 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" +) + +// FPutObject - Create an object in a bucket, with contents from file at filePath +func (c Client) FPutObject(bucketName, objectName, filePath string, opts PutObjectOptions) (n int64, err error) { + return c.FPutObjectWithContext(context.Background(), bucketName, objectName, filePath, opts) +} diff --git a/api-put-object-multipart.go b/api-put-object-multipart.go new file mode 100644 index 0000000..db92520 --- /dev/null +++ b/api-put-object-multipart.go @@ -0,0 +1,372 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "runtime/debug" + "sort" + "strconv" + "strings" + + "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/pkg/s3utils" +) + +func (c Client) putObjectMultipart(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, + opts PutObjectOptions) (n int64, err error) { + n, err = c.putObjectMultipartNoStream(ctx, bucketName, objectName, reader, opts) + if err != nil { + errResp := ToErrorResponse(err) + // Verify if multipart functionality is not available, if not + // fall back to single PutObject operation. + if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { + // Verify if size of reader is greater than '5GiB'. + if size > maxSinglePutObjectSize { + return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) + } + // Fall back to uploading as single PutObject operation. + return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + } + } + return n, err +} + +func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions) (n int64, err error) { + // Input validation. + if err = s3utils.CheckValidBucketName(bucketName); err != nil { + return 0, err + } + if err = s3utils.CheckValidObjectName(objectName); err != nil { + return 0, err + } + + // Total data read and written to server. should be equal to + // 'size' at the end of the call. + var totalUploadedSize int64 + + // Complete multipart upload. + var complMultipartUpload completeMultipartUpload + + // Calculate the optimal parts info for a given size. + totalPartsCount, partSize, _, err := optimalPartInfo(-1) + if err != nil { + return 0, err + } + + // Initiate a new multipart upload. + uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) + if err != nil { + return 0, err + } + + defer func() { + if err != nil { + c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) + } + }() + + // Part number always starts with '1'. + partNumber := 1 + + // Initialize parts uploaded map. + partsInfo := make(map[int]ObjectPart) + + // Create a buffer. + buf := make([]byte, partSize) + defer debug.FreeOSMemory() + + for partNumber <= totalPartsCount { + // Choose hash algorithms to be calculated by hashCopyN, + // avoid sha256 with non-v4 signature request or + // HTTPS connection. + hashAlgos, hashSums := c.hashMaterials() + + length, rErr := io.ReadFull(reader, buf) + if rErr == io.EOF { + break + } + if rErr != nil && rErr != io.ErrUnexpectedEOF { + return 0, rErr + } + + // Calculates hash sums while copying partSize bytes into cw. + for k, v := range hashAlgos { + v.Write(buf[:length]) + hashSums[k] = v.Sum(nil) + } + + // Update progress reader appropriately to the latest offset + // as we read from the source. + rd := newHook(bytes.NewReader(buf[:length]), opts.Progress) + + // Checksums.. + var ( + md5Base64 string + sha256Hex string + ) + if hashSums["md5"] != nil { + md5Base64 = base64.StdEncoding.EncodeToString(hashSums["md5"]) + } + if hashSums["sha256"] != nil { + sha256Hex = hex.EncodeToString(hashSums["sha256"]) + } + + // Proceed to upload the part. + var objPart ObjectPart + objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, + md5Base64, sha256Hex, int64(length), opts.ServerSideEncryption) + if err != nil { + return totalUploadedSize, err + } + + // Save successfully uploaded part metadata. + partsInfo[partNumber] = objPart + + // Save successfully uploaded size. + totalUploadedSize += int64(length) + + // Increment part number. + partNumber++ + + // For unknown size, Read EOF we break away. + // We do not have to upload till totalPartsCount. + if rErr == io.EOF { + break + } + } + + // Loop over total uploaded parts to save them in + // Parts array before completing the multipart request. + for i := 1; i < partNumber; i++ { + part, ok := partsInfo[i] + if !ok { + return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + } + complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + }) + } + + // Sort all completed parts. + sort.Sort(completedParts(complMultipartUpload.Parts)) + if _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload); err != nil { + return totalUploadedSize, err + } + + // Return final size. + return totalUploadedSize, nil +} + +// initiateMultipartUpload - Initiates a multipart upload and returns an upload ID. +func (c Client) initiateMultipartUpload(ctx context.Context, bucketName, objectName string, opts PutObjectOptions) (initiateMultipartUploadResult, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return initiateMultipartUploadResult{}, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return initiateMultipartUploadResult{}, err + } + + // Initialize url queries. + urlValues := make(url.Values) + urlValues.Set("uploads", "") + + // Set ContentType header. + customHeader := opts.Header() + + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + customHeader: customHeader, + } + + // Execute POST on an objectName to initiate multipart upload. + resp, err := c.executeMethod(ctx, "POST", reqMetadata) + defer closeResponse(resp) + if err != nil { + return initiateMultipartUploadResult{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return initiateMultipartUploadResult{}, httpRespToErrorResponse(resp, bucketName, objectName) + } + } + // Decode xml for new multipart upload. + initiateMultipartUploadResult := initiateMultipartUploadResult{} + err = xmlDecoder(resp.Body, &initiateMultipartUploadResult) + if err != nil { + return initiateMultipartUploadResult, err + } + return initiateMultipartUploadResult, nil +} + +// uploadPart - Uploads a part in a multipart upload. +func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID string, reader io.Reader, + partNumber int, md5Base64, sha256Hex string, size int64, sse encrypt.ServerSide) (ObjectPart, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return ObjectPart{}, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return ObjectPart{}, err + } + if size > maxPartSize { + return ObjectPart{}, ErrEntityTooLarge(size, maxPartSize, bucketName, objectName) + } + if size <= -1 { + return ObjectPart{}, ErrEntityTooSmall(size, bucketName, objectName) + } + if partNumber <= 0 { + return ObjectPart{}, ErrInvalidArgument("Part number cannot be negative or equal to zero.") + } + if uploadID == "" { + return ObjectPart{}, ErrInvalidArgument("UploadID cannot be empty.") + } + + // Get resources properly escaped and lined up before using them in http request. + urlValues := make(url.Values) + // Set part number. + urlValues.Set("partNumber", strconv.Itoa(partNumber)) + // Set upload id. + urlValues.Set("uploadId", uploadID) + + // Set encryption headers, if any. + customHeader := make(http.Header) + // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html + // Server-side encryption is supported by the S3 Multipart Upload actions. + // Unless you are using a customer-provided encryption key, you don't need + // to specify the encryption parameters in each UploadPart request. + if sse != nil && sse.Type() == encrypt.SSEC { + sse.Marshal(customHeader) + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + customHeader: customHeader, + contentBody: reader, + contentLength: size, + contentMD5Base64: md5Base64, + contentSHA256Hex: sha256Hex, + } + + // Execute PUT on each part. + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return ObjectPart{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return ObjectPart{}, httpRespToErrorResponse(resp, bucketName, objectName) + } + } + // Once successfully uploaded, return completed part. + objPart := ObjectPart{} + objPart.Size = size + objPart.PartNumber = partNumber + // Trim off the odd double quotes from ETag in the beginning and end. + objPart.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"") + objPart.ETag = strings.TrimSuffix(objPart.ETag, "\"") + return objPart, nil +} + +// completeMultipartUpload - Completes a multipart upload by assembling previously uploaded parts. +func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string, + complete completeMultipartUpload) (completeMultipartUploadResult, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return completeMultipartUploadResult{}, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return completeMultipartUploadResult{}, err + } + + // Initialize url queries. + urlValues := make(url.Values) + urlValues.Set("uploadId", uploadID) + // Marshal complete multipart body. + completeMultipartUploadBytes, err := xml.Marshal(complete) + if err != nil { + return completeMultipartUploadResult{}, err + } + + // Instantiate all the complete multipart buffer. + completeMultipartUploadBuffer := bytes.NewReader(completeMultipartUploadBytes) + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentBody: completeMultipartUploadBuffer, + contentLength: int64(len(completeMultipartUploadBytes)), + contentSHA256Hex: sum256Hex(completeMultipartUploadBytes), + } + + // Execute POST to complete multipart upload for an objectName. + resp, err := c.executeMethod(ctx, "POST", reqMetadata) + defer closeResponse(resp) + if err != nil { + return completeMultipartUploadResult{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return completeMultipartUploadResult{}, httpRespToErrorResponse(resp, bucketName, objectName) + } + } + + // Read resp.Body into a []bytes to parse for Error response inside the body + var b []byte + b, err = ioutil.ReadAll(resp.Body) + if err != nil { + return completeMultipartUploadResult{}, err + } + // Decode completed multipart upload response on success. + completeMultipartUploadResult := completeMultipartUploadResult{} + err = xmlDecoder(bytes.NewReader(b), &completeMultipartUploadResult) + if err != nil { + // xml parsing failure due to presence an ill-formed xml fragment + return completeMultipartUploadResult, err + } else if completeMultipartUploadResult.Bucket == "" { + // xml's Decode method ignores well-formed xml that don't apply to the type of value supplied. + // In this case, it would leave completeMultipartUploadResult with the corresponding zero-values + // of the members. + + // Decode completed multipart upload response on failure + completeMultipartUploadErr := ErrorResponse{} + err = xmlDecoder(bytes.NewReader(b), &completeMultipartUploadErr) + if err != nil { + // xml parsing failure due to presence an ill-formed xml fragment + return completeMultipartUploadResult, err + } + return completeMultipartUploadResult, completeMultipartUploadErr + } + return completeMultipartUploadResult, nil +} diff --git a/api-put-object-streaming.go b/api-put-object-streaming.go new file mode 100644 index 0000000..211d1c2 --- /dev/null +++ b/api-put-object-streaming.go @@ -0,0 +1,417 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "fmt" + "io" + "net/http" + "sort" + "strings" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// putObjectMultipartStream - upload a large object using +// multipart upload and streaming signature for signing payload. +// Comprehensive put object operation involving multipart uploads. +// +// Following code handles these types of readers. +// +// - *minio.Object +// - Any reader which has a method 'ReadAt()' +// +func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, objectName string, + reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { + + if !isObject(reader) && isReadAt(reader) { + // Verify if the reader implements ReadAt and it is not a *minio.Object then we will use parallel uploader. + n, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts) + } else { + n, err = c.putObjectMultipartStreamNoChecksum(ctx, bucketName, objectName, reader, size, opts) + } + if err != nil { + errResp := ToErrorResponse(err) + // Verify if multipart functionality is not available, if not + // fall back to single PutObject operation. + if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { + // Verify if size of reader is greater than '5GiB'. + if size > maxSinglePutObjectSize { + return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) + } + // Fall back to uploading as single PutObject operation. + return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + } + } + return n, err +} + +// uploadedPartRes - the response received from a part upload. +type uploadedPartRes struct { + Error error // Any error encountered while uploading the part. + PartNum int // Number of the part uploaded. + Size int64 // Size of the part uploaded. + Part *ObjectPart +} + +type uploadPartReq struct { + PartNum int // Number of the part uploaded. + Part *ObjectPart // Size of the part uploaded. +} + +// putObjectMultipartFromReadAt - Uploads files bigger than 64MiB. +// Supports all readers which implements io.ReaderAt interface +// (ReadAt method). +// +// NOTE: This function is meant to be used for all readers which +// implement io.ReaderAt which allows us for resuming multipart +// uploads but reading at an offset, which would avoid re-read the +// data which was already uploaded. Internally this function uses +// temporary files for staging all the data, these temporary files are +// cleaned automatically when the caller i.e http client closes the +// stream after uploading all the contents successfully. +func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketName, objectName string, + reader io.ReaderAt, size int64, opts PutObjectOptions) (n int64, err error) { + // Input validation. + if err = s3utils.CheckValidBucketName(bucketName); err != nil { + return 0, err + } + if err = s3utils.CheckValidObjectName(objectName); err != nil { + return 0, err + } + + // Calculate the optimal parts info for a given size. + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size) + if err != nil { + return 0, err + } + + // Initiate a new multipart upload. + uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) + if err != nil { + return 0, err + } + + // Aborts the multipart upload in progress, if the + // function returns any error, since we do not resume + // we should purge the parts which have been uploaded + // to relinquish storage space. + defer func() { + if err != nil { + c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) + } + }() + + // Total data read and written to server. should be equal to 'size' at the end of the call. + var totalUploadedSize int64 + + // Complete multipart upload. + var complMultipartUpload completeMultipartUpload + + // Declare a channel that sends the next part number to be uploaded. + // Buffered to 10000 because thats the maximum number of parts allowed + // by S3. + uploadPartsCh := make(chan uploadPartReq, 10000) + + // Declare a channel that sends back the response of a part upload. + // Buffered to 10000 because thats the maximum number of parts allowed + // by S3. + uploadedPartsCh := make(chan uploadedPartRes, 10000) + + // Used for readability, lastPartNumber is always totalPartsCount. + lastPartNumber := totalPartsCount + + // Send each part number to the channel to be processed. + for p := 1; p <= totalPartsCount; p++ { + uploadPartsCh <- uploadPartReq{PartNum: p, Part: nil} + } + close(uploadPartsCh) + // Receive each part number from the channel allowing three parallel uploads. + for w := 1; w <= opts.getNumThreads(); w++ { + go func(partSize int64) { + // Each worker will draw from the part channel and upload in parallel. + for uploadReq := range uploadPartsCh { + + // If partNumber was not uploaded we calculate the missing + // part offset and size. For all other part numbers we + // calculate offset based on multiples of partSize. + readOffset := int64(uploadReq.PartNum-1) * partSize + + // As a special case if partNumber is lastPartNumber, we + // calculate the offset based on the last part size. + if uploadReq.PartNum == lastPartNumber { + readOffset = (size - lastPartSize) + partSize = lastPartSize + } + + // Get a section reader on a particular offset. + sectionReader := newHook(io.NewSectionReader(reader, readOffset, partSize), opts.Progress) + + // Proceed to upload the part. + var objPart ObjectPart + objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, + sectionReader, uploadReq.PartNum, + "", "", partSize, opts.ServerSideEncryption) + if err != nil { + uploadedPartsCh <- uploadedPartRes{ + Size: 0, + Error: err, + } + // Exit the goroutine. + return + } + + // Save successfully uploaded part metadata. + uploadReq.Part = &objPart + + // Send successful part info through the channel. + uploadedPartsCh <- uploadedPartRes{ + Size: objPart.Size, + PartNum: uploadReq.PartNum, + Part: uploadReq.Part, + Error: nil, + } + } + }(partSize) + } + + // Gather the responses as they occur and update any + // progress bar. + for u := 1; u <= totalPartsCount; u++ { + uploadRes := <-uploadedPartsCh + if uploadRes.Error != nil { + return totalUploadedSize, uploadRes.Error + } + // Retrieve each uploaded part and store it to be completed. + // part, ok := partsInfo[uploadRes.PartNum] + part := uploadRes.Part + if part == nil { + return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", uploadRes.PartNum)) + } + // Update the totalUploadedSize. + totalUploadedSize += uploadRes.Size + // Store the parts to be completed in order. + complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + }) + } + + // Verify if we uploaded all the data. + if totalUploadedSize != size { + return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) + } + + // Sort all completed parts. + sort.Sort(completedParts(complMultipartUpload.Parts)) + _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + if err != nil { + return totalUploadedSize, err + } + + // Return final size. + return totalUploadedSize, nil +} + +func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketName, objectName string, + reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { + // Input validation. + if err = s3utils.CheckValidBucketName(bucketName); err != nil { + return 0, err + } + if err = s3utils.CheckValidObjectName(objectName); err != nil { + return 0, err + } + + // Calculate the optimal parts info for a given size. + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size) + if err != nil { + return 0, err + } + // Initiates a new multipart request + uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) + if err != nil { + return 0, err + } + + // Aborts the multipart upload if the function returns + // any error, since we do not resume we should purge + // the parts which have been uploaded to relinquish + // storage space. + defer func() { + if err != nil { + c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) + } + }() + + // Total data read and written to server. should be equal to 'size' at the end of the call. + var totalUploadedSize int64 + + // Initialize parts uploaded map. + partsInfo := make(map[int]ObjectPart) + + // Part number always starts with '1'. + var partNumber int + for partNumber = 1; partNumber <= totalPartsCount; partNumber++ { + // Update progress reader appropriately to the latest offset + // as we read from the source. + hookReader := newHook(reader, opts.Progress) + + // Proceed to upload the part. + if partNumber == totalPartsCount { + partSize = lastPartSize + } + var objPart ObjectPart + objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, + io.LimitReader(hookReader, partSize), + partNumber, "", "", partSize, opts.ServerSideEncryption) + if err != nil { + return totalUploadedSize, err + } + + // Save successfully uploaded part metadata. + partsInfo[partNumber] = objPart + + // Save successfully uploaded size. + totalUploadedSize += partSize + } + + // Verify if we uploaded all the data. + if size > 0 { + if totalUploadedSize != size { + return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) + } + } + + // Complete multipart upload. + var complMultipartUpload completeMultipartUpload + + // Loop over total uploaded parts to save them in + // Parts array before completing the multipart request. + for i := 1; i < partNumber; i++ { + part, ok := partsInfo[i] + if !ok { + return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + } + complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + }) + } + + // Sort all completed parts. + sort.Sort(completedParts(complMultipartUpload.Parts)) + _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + if err != nil { + return totalUploadedSize, err + } + + // Return final size. + return totalUploadedSize, nil +} + +// putObjectNoChecksum special function used Google Cloud Storage. This special function +// is used for Google Cloud Storage since Google's multipart API is not S3 compatible. +func (c Client) putObjectNoChecksum(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return 0, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return 0, err + } + + // Size -1 is only supported on Google Cloud Storage, we error + // out in all other situations. + if size < 0 && !s3utils.IsGoogleEndpoint(*c.endpointURL) { + return 0, ErrEntityTooSmall(size, bucketName, objectName) + } + if size > 0 { + if isReadAt(reader) && !isObject(reader) { + seeker, _ := reader.(io.Seeker) + offset, err := seeker.Seek(0, io.SeekCurrent) + if err != nil { + return 0, ErrInvalidArgument(err.Error()) + } + reader = io.NewSectionReader(reader.(io.ReaderAt), offset, size) + } + } + + // Update progress reader appropriately to the latest offset as we + // read from the source. + readSeeker := newHook(reader, opts.Progress) + + // This function does not calculate sha256 and md5sum for payload. + // Execute put object. + st, err := c.putObjectDo(ctx, bucketName, objectName, readSeeker, "", "", size, opts) + if err != nil { + return 0, err + } + if st.Size != size { + return 0, ErrUnexpectedEOF(st.Size, size, bucketName, objectName) + } + return size, nil +} + +// putObjectDo - executes the put object http operation. +// NOTE: You must have WRITE permissions on a bucket to add an object to it. +func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, reader io.Reader, md5Base64, sha256Hex string, size int64, opts PutObjectOptions) (ObjectInfo, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return ObjectInfo{}, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return ObjectInfo{}, err + } + // Set headers. + customHeader := opts.Header() + + // Populate request metadata. + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + customHeader: customHeader, + contentBody: reader, + contentLength: size, + contentMD5Base64: md5Base64, + contentSHA256Hex: sha256Hex, + } + + // Execute PUT an objectName. + resp, err := c.executeMethod(ctx, "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return ObjectInfo{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) + } + } + + var objInfo ObjectInfo + // Trim off the odd double quotes from ETag in the beginning and end. + objInfo.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"") + objInfo.ETag = strings.TrimSuffix(objInfo.ETag, "\"") + // A success here means data was written to server successfully. + objInfo.Size = size + + // Return here. + return objInfo, nil +} diff --git a/api-put-object.go b/api-put-object.go new file mode 100644 index 0000000..0330cd9 --- /dev/null +++ b/api-put-object.go @@ -0,0 +1,267 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "runtime/debug" + "sort" + + "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/pkg/s3utils" + "golang.org/x/net/http/httpguts" +) + +// PutObjectOptions represents options specified by user for PutObject call +type PutObjectOptions struct { + UserMetadata map[string]string + Progress io.Reader + ContentType string + ContentEncoding string + ContentDisposition string + ContentLanguage string + CacheControl string + ServerSideEncryption encrypt.ServerSide + NumThreads uint + StorageClass string + WebsiteRedirectLocation string +} + +// getNumThreads - gets the number of threads to be used in the multipart +// put object operation +func (opts PutObjectOptions) getNumThreads() (numThreads int) { + if opts.NumThreads > 0 { + numThreads = int(opts.NumThreads) + } else { + numThreads = totalWorkers + } + return +} + +// Header - constructs the headers from metadata entered by user in +// PutObjectOptions struct +func (opts PutObjectOptions) Header() (header http.Header) { + header = make(http.Header) + + if opts.ContentType != "" { + header["Content-Type"] = []string{opts.ContentType} + } else { + header["Content-Type"] = []string{"application/octet-stream"} + } + if opts.ContentEncoding != "" { + header["Content-Encoding"] = []string{opts.ContentEncoding} + } + if opts.ContentDisposition != "" { + header["Content-Disposition"] = []string{opts.ContentDisposition} + } + if opts.ContentLanguage != "" { + header["Content-Language"] = []string{opts.ContentLanguage} + } + if opts.CacheControl != "" { + header["Cache-Control"] = []string{opts.CacheControl} + } + if opts.ServerSideEncryption != nil { + opts.ServerSideEncryption.Marshal(header) + } + if opts.StorageClass != "" { + header[amzStorageClass] = []string{opts.StorageClass} + } + if opts.WebsiteRedirectLocation != "" { + header[amzWebsiteRedirectLocation] = []string{opts.WebsiteRedirectLocation} + } + for k, v := range opts.UserMetadata { + if !isAmzHeader(k) && !isStandardHeader(k) && !isStorageClassHeader(k) { + header["X-Amz-Meta-"+k] = []string{v} + } else { + header[k] = []string{v} + } + } + return +} + +// validate() checks if the UserMetadata map has standard headers or and raises an error if so. +func (opts PutObjectOptions) validate() (err error) { + for k, v := range opts.UserMetadata { + if !httpguts.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) { + return ErrInvalidArgument(k + " unsupported user defined metadata name") + } + if !httpguts.ValidHeaderFieldValue(v) { + return ErrInvalidArgument(v + " unsupported user defined metadata value") + } + } + return nil +} + +// completedParts is a collection of parts sortable by their part numbers. +// used for sorting the uploaded parts before completing the multipart request. +type completedParts []CompletePart + +func (a completedParts) Len() int { return len(a) } +func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber } + +// PutObject creates an object in a bucket. +// +// You must have WRITE permissions on a bucket to create an object. +// +// - For size smaller than 64MiB PutObject automatically does a +// single atomic Put operation. +// - For size larger than 64MiB PutObject automatically does a +// multipart Put operation. +// - For size input as -1 PutObject does a multipart Put operation +// until input stream reaches EOF. Maximum object size that can +// be uploaded through this operation will be 5TiB. +func (c Client) PutObject(bucketName, objectName string, reader io.Reader, objectSize int64, + opts PutObjectOptions) (n int64, err error) { + return c.PutObjectWithContext(context.Background(), bucketName, objectName, reader, objectSize, opts) +} + +func (c Client) putObjectCommon(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) { + // Check for largest object size allowed. + if size > int64(maxMultipartPutObjectSize) { + return 0, ErrEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName) + } + + // NOTE: Streaming signature is not supported by GCS. + if s3utils.IsGoogleEndpoint(*c.endpointURL) { + // Do not compute MD5 for Google Cloud Storage. + return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + } + + if c.overrideSignerType.IsV2() { + if size >= 0 && size < minPartSize { + return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + } + return c.putObjectMultipart(ctx, bucketName, objectName, reader, size, opts) + } + if size < 0 { + return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts) + } + + if size < minPartSize { + return c.putObjectNoChecksum(ctx, bucketName, objectName, reader, size, opts) + } + // For all sizes greater than 64MiB do multipart. + return c.putObjectMultipartStream(ctx, bucketName, objectName, reader, size, opts) +} + +func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions) (n int64, err error) { + // Input validation. + if err = s3utils.CheckValidBucketName(bucketName); err != nil { + return 0, err + } + if err = s3utils.CheckValidObjectName(objectName); err != nil { + return 0, err + } + + // Total data read and written to server. should be equal to + // 'size' at the end of the call. + var totalUploadedSize int64 + + // Complete multipart upload. + var complMultipartUpload completeMultipartUpload + + // Calculate the optimal parts info for a given size. + totalPartsCount, partSize, _, err := optimalPartInfo(-1) + if err != nil { + return 0, err + } + // Initiate a new multipart upload. + uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) + if err != nil { + return 0, err + } + + defer func() { + if err != nil { + c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) + } + }() + + // Part number always starts with '1'. + partNumber := 1 + + // Initialize parts uploaded map. + partsInfo := make(map[int]ObjectPart) + + // Create a buffer. + buf := make([]byte, partSize) + defer debug.FreeOSMemory() + + for partNumber <= totalPartsCount { + length, rErr := io.ReadFull(reader, buf) + if rErr == io.EOF && partNumber > 1 { + break + } + if rErr != nil && rErr != io.ErrUnexpectedEOF && rErr != io.EOF { + return 0, rErr + } + // Update progress reader appropriately to the latest offset + // as we read from the source. + rd := newHook(bytes.NewReader(buf[:length]), opts.Progress) + + // Proceed to upload the part. + var objPart ObjectPart + objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, + "", "", int64(length), opts.ServerSideEncryption) + if err != nil { + return totalUploadedSize, err + } + + // Save successfully uploaded part metadata. + partsInfo[partNumber] = objPart + + // Save successfully uploaded size. + totalUploadedSize += int64(length) + + // Increment part number. + partNumber++ + + // For unknown size, Read EOF we break away. + // We do not have to upload till totalPartsCount. + if rErr == io.EOF { + break + } + } + + // Loop over total uploaded parts to save them in + // Parts array before completing the multipart request. + for i := 1; i < partNumber; i++ { + part, ok := partsInfo[i] + if !ok { + return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + } + complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + }) + } + + // Sort all completed parts. + sort.Sort(completedParts(complMultipartUpload.Parts)) + if _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload); err != nil { + return totalUploadedSize, err + } + + // Return final size. + return totalUploadedSize, nil +} diff --git a/api-put-object_test.go b/api-put-object_test.go new file mode 100644 index 0000000..d96abab --- /dev/null +++ b/api-put-object_test.go @@ -0,0 +1,63 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package minio + +import ( + "testing" +) + +func TestPutObjectOptionsValidate(t *testing.T) { + testCases := []struct { + name, value string + shouldPass bool + }{ + // Invalid cases. + {"X-Amz-Matdesc", "blah", false}, + {"x-amz-meta-X-Amz-Iv", "blah", false}, + {"x-amz-meta-X-Amz-Key", "blah", false}, + {"x-amz-meta-X-Amz-Matdesc", "blah", false}, + {"It has spaces", "v", false}, + {"It,has@illegal=characters", "v", false}, + {"X-Amz-Iv", "blah", false}, + {"X-Amz-Key", "blah", false}, + {"X-Amz-Key-prefixed-header", "blah", false}, + {"Content-Type", "custom/content-type", false}, + {"content-type", "custom/content-type", false}, + {"Content-Encoding", "gzip", false}, + {"Cache-Control", "blah", false}, + {"Content-Disposition", "something", false}, + {"Content-Language", "somelanguage", false}, + + // Valid metadata names. + {"my-custom-header", "blah", true}, + {"custom-X-Amz-Key-middle", "blah", true}, + {"my-custom-header-X-Amz-Key", "blah", true}, + {"blah-X-Amz-Matdesc", "blah", true}, + {"X-Amz-MatDesc-suffix", "blah", true}, + {"It-Is-Fine", "v", true}, + {"Numbers-098987987-Should-Work", "v", true}, + {"Crazy-!#$%&'*+-.^_`|~-Should-193832-Be-Fine", "v", true}, + } + for i, testCase := range testCases { + err := PutObjectOptions{UserMetadata: map[string]string{ + testCase.name: testCase.value, + }}.validate() + if testCase.shouldPass && err != nil { + t.Errorf("Test %d - output did not match with reference results, %s", i+1, err) + } + } +} diff --git a/api-remove.go b/api-remove.go new file mode 100644 index 0000000..f33df4d --- /dev/null +++ b/api-remove.go @@ -0,0 +1,303 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "io" + "net/http" + "net/url" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// RemoveBucket deletes the bucket name. +// +// All objects (including all object versions and delete markers). +// in the bucket must be deleted before successfully attempting this request. +func (c Client) RemoveBucket(bucketName string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + // Execute DELETE on bucket. + resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ + bucketName: bucketName, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, "") + } + } + + // Remove the location from cache on a successful delete. + c.bucketLocCache.Delete(bucketName) + + return nil +} + +// RemoveObject remove an object from a bucket. +func (c Client) RemoveObject(bucketName, objectName string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return err + } + // Execute DELETE on objectName. + resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + // if some unexpected error happened and max retry is reached, we want to let client know + if resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, objectName) + } + } + + // DeleteObject always responds with http '204' even for + // objects which do not exist. So no need to handle them + // specifically. + return nil +} + +// RemoveObjectError - container of Multi Delete S3 API error +type RemoveObjectError struct { + ObjectName string + Err error +} + +// generateRemoveMultiObjects - generate the XML request for remove multi objects request +func generateRemoveMultiObjectsRequest(objects []string) []byte { + rmObjects := []deleteObject{} + for _, obj := range objects { + rmObjects = append(rmObjects, deleteObject{Key: obj}) + } + xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: rmObjects, Quiet: true}) + return xmlBytes +} + +// processRemoveMultiObjectsResponse - parse the remove multi objects web service +// and return the success/failure result status for each object +func processRemoveMultiObjectsResponse(body io.Reader, objects []string, errorCh chan<- RemoveObjectError) { + // Parse multi delete XML response + rmResult := &deleteMultiObjectsResult{} + err := xmlDecoder(body, rmResult) + if err != nil { + errorCh <- RemoveObjectError{ObjectName: "", Err: err} + return + } + + // Fill deletion that returned an error. + for _, obj := range rmResult.UnDeletedObjects { + errorCh <- RemoveObjectError{ + ObjectName: obj.Key, + Err: ErrorResponse{ + Code: obj.Code, + Message: obj.Message, + }, + } + } +} + +// RemoveObjectsWithContext - Identical to RemoveObjects call, but accepts context to facilitate request cancellation. +func (c Client) RemoveObjectsWithContext(ctx context.Context, bucketName string, objectsCh <-chan string) <-chan RemoveObjectError { + errorCh := make(chan RemoveObjectError, 1) + + // Validate if bucket name is valid. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + defer close(errorCh) + errorCh <- RemoveObjectError{ + Err: err, + } + return errorCh + } + // Validate objects channel to be properly allocated. + if objectsCh == nil { + defer close(errorCh) + errorCh <- RemoveObjectError{ + Err: ErrInvalidArgument("Objects channel cannot be nil"), + } + return errorCh + } + + // Generate and call MultiDelete S3 requests based on entries received from objectsCh + go func(errorCh chan<- RemoveObjectError) { + maxEntries := 1000 + finish := false + urlValues := make(url.Values) + urlValues.Set("delete", "") + + // Close error channel when Multi delete finishes. + defer close(errorCh) + + // Loop over entries by 1000 and call MultiDelete requests + for { + if finish { + break + } + count := 0 + var batch []string + + // Try to gather 1000 entries + for object := range objectsCh { + batch = append(batch, object) + if count++; count >= maxEntries { + break + } + } + if count == 0 { + // Multi Objects Delete API doesn't accept empty object list, quit immediately + break + } + if count < maxEntries { + // We didn't have 1000 entries, so this is the last batch + finish = true + } + + // Generate remove multi objects XML request + removeBytes := generateRemoveMultiObjectsRequest(batch) + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(ctx, "POST", requestMetadata{ + bucketName: bucketName, + queryValues: urlValues, + contentBody: bytes.NewReader(removeBytes), + contentLength: int64(len(removeBytes)), + contentMD5Base64: sumMD5Base64(removeBytes), + contentSHA256Hex: sum256Hex(removeBytes), + }) + if resp != nil { + if resp.StatusCode != http.StatusOK { + e := httpRespToErrorResponse(resp, bucketName, "") + errorCh <- RemoveObjectError{ObjectName: "", Err: e} + } + } + if err != nil { + for _, b := range batch { + errorCh <- RemoveObjectError{ObjectName: b, Err: err} + } + continue + } + + // Process multiobjects remove xml response + processRemoveMultiObjectsResponse(resp.Body, batch, errorCh) + + closeResponse(resp) + } + }(errorCh) + return errorCh +} + +// RemoveObjects removes multiple objects from a bucket. +// The list of objects to remove are received from objectsCh. +// Remove failures are sent back via error channel. +func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan RemoveObjectError { + return c.RemoveObjectsWithContext(context.Background(), bucketName, objectsCh) +} + +// RemoveIncompleteUpload aborts an partially uploaded object. +func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return err + } + // Find multipart upload ids of the object to be aborted. + uploadIDs, err := c.findUploadIDs(bucketName, objectName) + if err != nil { + return err + } + + for _, uploadID := range uploadIDs { + // abort incomplete multipart upload, based on the upload id passed. + err := c.abortMultipartUpload(context.Background(), bucketName, objectName, uploadID) + if err != nil { + return err + } + } + + return nil +} + +// abortMultipartUpload aborts a multipart upload for the given +// uploadID, all previously uploaded parts are deleted. +func (c Client) abortMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return err + } + + // Initialize url queries. + urlValues := make(url.Values) + urlValues.Set("uploadId", uploadID) + + // Execute DELETE on multipart upload. + resp, err := c.executeMethod(ctx, "DELETE", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusNoContent { + // Abort has no response body, handle it for any errors. + var errorResponse ErrorResponse + switch resp.StatusCode { + case http.StatusNotFound: + // This is needed specifically for abort and it cannot + // be converged into default case. + errorResponse = ErrorResponse{ + Code: "NoSuchUpload", + Message: "The specified multipart upload does not exist.", + BucketName: bucketName, + Key: objectName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), + } + default: + return httpRespToErrorResponse(resp, bucketName, objectName) + } + return errorResponse + } + } + return nil +} diff --git a/api-s3-datatypes.go b/api-s3-datatypes.go new file mode 100644 index 0000000..8d8880c --- /dev/null +++ b/api-s3-datatypes.go @@ -0,0 +1,245 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "encoding/xml" + "time" +) + +// listAllMyBucketsResult container for listBuckets response. +type listAllMyBucketsResult struct { + // Container for one or more buckets. + Buckets struct { + Bucket []BucketInfo + } + Owner owner +} + +// owner container for bucket owner information. +type owner struct { + DisplayName string + ID string +} + +// CommonPrefix container for prefix response. +type CommonPrefix struct { + Prefix string +} + +// ListBucketV2Result container for listObjects response version 2. +type ListBucketV2Result struct { + // A response can contain CommonPrefixes only if you have + // specified a delimiter. + CommonPrefixes []CommonPrefix + // Metadata about each object returned. + Contents []ObjectInfo + Delimiter string + + // Encoding type used to encode object keys in the response. + EncodingType string + + // A flag that indicates whether or not ListObjects returned all of the results + // that satisfied the search criteria. + IsTruncated bool + MaxKeys int64 + Name string + + // Hold the token that will be sent in the next request to fetch the next group of keys + NextContinuationToken string + + ContinuationToken string + Prefix string + + // FetchOwner and StartAfter are currently not used + FetchOwner string + StartAfter string +} + +// ListBucketResult container for listObjects response. +type ListBucketResult struct { + // A response can contain CommonPrefixes only if you have + // specified a delimiter. + CommonPrefixes []CommonPrefix + // Metadata about each object returned. + Contents []ObjectInfo + Delimiter string + + // Encoding type used to encode object keys in the response. + EncodingType string + + // A flag that indicates whether or not ListObjects returned all of the results + // that satisfied the search criteria. + IsTruncated bool + Marker string + MaxKeys int64 + Name string + + // When response is truncated (the IsTruncated element value in + // the response is true), you can use the key name in this field + // as marker in the subsequent request to get next set of objects. + // Object storage lists objects in alphabetical order Note: This + // element is returned only if you have delimiter request + // parameter specified. If response does not include the NextMaker + // and it is truncated, you can use the value of the last Key in + // the response as the marker in the subsequent request to get the + // next set of object keys. + NextMarker string + Prefix string +} + +// ListMultipartUploadsResult container for ListMultipartUploads response +type ListMultipartUploadsResult struct { + Bucket string + KeyMarker string + UploadIDMarker string `xml:"UploadIdMarker"` + NextKeyMarker string + NextUploadIDMarker string `xml:"NextUploadIdMarker"` + EncodingType string + MaxUploads int64 + IsTruncated bool + Uploads []ObjectMultipartInfo `xml:"Upload"` + Prefix string + Delimiter string + // A response can contain CommonPrefixes only if you specify a delimiter. + CommonPrefixes []CommonPrefix +} + +// initiator container for who initiated multipart upload. +type initiator struct { + ID string + DisplayName string +} + +// copyObjectResult container for copy object response. +type copyObjectResult struct { + ETag string + LastModified time.Time // time string format "2006-01-02T15:04:05.000Z" +} + +// ObjectPart container for particular part of an object. +type ObjectPart struct { + // Part number identifies the part. + PartNumber int + + // Date and time the part was uploaded. + LastModified time.Time + + // Entity tag returned when the part was uploaded, usually md5sum + // of the part. + ETag string + + // Size of the uploaded part data. + Size int64 +} + +// ListObjectPartsResult container for ListObjectParts response. +type ListObjectPartsResult struct { + Bucket string + Key string + UploadID string `xml:"UploadId"` + + Initiator initiator + Owner owner + + StorageClass string + PartNumberMarker int + NextPartNumberMarker int + MaxParts int + + // Indicates whether the returned list of parts is truncated. + IsTruncated bool + ObjectParts []ObjectPart `xml:"Part"` + + EncodingType string +} + +// initiateMultipartUploadResult container for InitiateMultiPartUpload +// response. +type initiateMultipartUploadResult struct { + Bucket string + Key string + UploadID string `xml:"UploadId"` +} + +// completeMultipartUploadResult container for completed multipart +// upload response. +type completeMultipartUploadResult struct { + Location string + Bucket string + Key string + ETag string +} + +// CompletePart sub container lists individual part numbers and their +// md5sum, part of completeMultipartUpload. +type CompletePart struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Part" json:"-"` + + // Part number identifies the part. + PartNumber int + ETag string +} + +// completeMultipartUpload container for completing multipart upload. +type completeMultipartUpload struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUpload" json:"-"` + Parts []CompletePart `xml:"Part"` +} + +// createBucketConfiguration container for bucket configuration. +type createBucketConfiguration struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreateBucketConfiguration" json:"-"` + Location string `xml:"LocationConstraint"` +} + +// deleteObject container for Delete element in MultiObjects Delete XML request +type deleteObject struct { + Key string + VersionID string `xml:"VersionId,omitempty"` +} + +// deletedObject container for Deleted element in MultiObjects Delete XML response +type deletedObject struct { + Key string + VersionID string `xml:"VersionId,omitempty"` + // These fields are ignored. + DeleteMarker bool + DeleteMarkerVersionID string +} + +// nonDeletedObject container for Error element (failed deletion) in MultiObjects Delete XML response +type nonDeletedObject struct { + Key string + Code string + Message string +} + +// deletedMultiObjects container for MultiObjects Delete XML request +type deleteMultiObjects struct { + XMLName xml.Name `xml:"Delete"` + Quiet bool + Objects []deleteObject `xml:"Object"` +} + +// deletedMultiObjectsResult container for MultiObjects Delete XML response +type deleteMultiObjectsResult struct { + XMLName xml.Name `xml:"DeleteResult"` + DeletedObjects []deletedObject `xml:"Deleted"` + UnDeletedObjects []nonDeletedObject `xml:"Error"` +} diff --git a/api-select.go b/api-select.go new file mode 100644 index 0000000..a9b6f17 --- /dev/null +++ b/api-select.go @@ -0,0 +1,520 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/xml" + "errors" + "fmt" + "hash" + "hash/crc32" + "io" + "net/http" + "net/url" + "strings" + + "github.com/minio/minio-go/pkg/encrypt" + "github.com/minio/minio-go/pkg/s3utils" +) + +// CSVFileHeaderInfo - is the parameter for whether to utilize headers. +type CSVFileHeaderInfo string + +// Constants for file header info. +const ( + CSVFileHeaderInfoNone CSVFileHeaderInfo = "NONE" + CSVFileHeaderInfoIgnore = "IGNORE" + CSVFileHeaderInfoUse = "USE" +) + +// SelectCompressionType - is the parameter for what type of compression is +// present +type SelectCompressionType string + +// Constants for compression types under select API. +const ( + SelectCompressionNONE SelectCompressionType = "NONE" + SelectCompressionGZIP = "GZIP" + SelectCompressionBZIP = "BZIP2" +) + +// CSVQuoteFields - is the parameter for how CSV fields are quoted. +type CSVQuoteFields string + +// Constants for csv quote styles. +const ( + CSVQuoteFieldsAlways CSVQuoteFields = "Always" + CSVQuoteFieldsAsNeeded = "AsNeeded" +) + +// QueryExpressionType - is of what syntax the expression is, this should only +// be SQL +type QueryExpressionType string + +// Constants for expression type. +const ( + QueryExpressionTypeSQL QueryExpressionType = "SQL" +) + +// JSONType determines json input serialization type. +type JSONType string + +// Constants for JSONTypes. +const ( + JSONDocumentType JSONType = "DOCUMENT" + JSONLinesType = "LINES" +) + +// ParquetInputOptions parquet input specific options +type ParquetInputOptions struct{} + +// CSVInputOptions csv input specific options +type CSVInputOptions struct { + FileHeaderInfo CSVFileHeaderInfo + RecordDelimiter string + FieldDelimiter string + QuoteCharacter string + QuoteEscapeCharacter string + Comments string +} + +// CSVOutputOptions csv output specific options +type CSVOutputOptions struct { + QuoteFields CSVQuoteFields + RecordDelimiter string + FieldDelimiter string + QuoteCharacter string + QuoteEscapeCharacter string +} + +// JSONInputOptions json input specific options +type JSONInputOptions struct { + Type JSONType +} + +// JSONOutputOptions - json output specific options +type JSONOutputOptions struct { + RecordDelimiter string +} + +// SelectObjectInputSerialization - input serialization parameters +type SelectObjectInputSerialization struct { + CompressionType SelectCompressionType + Parquet *ParquetInputOptions `xml:"Parquet,omitempty"` + CSV *CSVInputOptions `xml:"CSV,omitempty"` + JSON *JSONInputOptions `xml:"JSON,omitempty"` +} + +// SelectObjectOutputSerialization - output serialization parameters. +type SelectObjectOutputSerialization struct { + CSV *CSVOutputOptions `xml:"CSV,omitempty"` + JSON *JSONOutputOptions `xml:"JSON,omitempty"` +} + +// SelectObjectOptions - represents the input select body +type SelectObjectOptions struct { + XMLName xml.Name `xml:"SelectObjectContentRequest" json:"-"` + ServerSideEncryption encrypt.ServerSide `xml:"-"` + Expression string + ExpressionType QueryExpressionType + InputSerialization SelectObjectInputSerialization + OutputSerialization SelectObjectOutputSerialization + RequestProgress struct { + Enabled bool + } +} + +// Header returns the http.Header representation of the SelectObject options. +func (o SelectObjectOptions) Header() http.Header { + headers := make(http.Header) + if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() == encrypt.SSEC { + o.ServerSideEncryption.Marshal(headers) + } + return headers +} + +// SelectObjectType - is the parameter which defines what type of object the +// operation is being performed on. +type SelectObjectType string + +// Constants for input data types. +const ( + SelectObjectTypeCSV SelectObjectType = "CSV" + SelectObjectTypeJSON = "JSON" + SelectObjectTypeParquet = "Parquet" +) + +// preludeInfo is used for keeping track of necessary information from the +// prelude. +type preludeInfo struct { + totalLen uint32 + headerLen uint32 +} + +// SelectResults is used for the streaming responses from the server. +type SelectResults struct { + pipeReader *io.PipeReader + resp *http.Response + stats *StatsMessage + progress *ProgressMessage +} + +// ProgressMessage is a struct for progress xml message. +type ProgressMessage struct { + XMLName xml.Name `xml:"Progress" json:"-"` + StatsMessage +} + +// StatsMessage is a struct for stat xml message. +type StatsMessage struct { + XMLName xml.Name `xml:"Stats" json:"-"` + BytesScanned int64 + BytesProcessed int64 + BytesReturned int64 +} + +// eventType represents the type of event. +type eventType string + +// list of event-types returned by Select API. +const ( + endEvent eventType = "End" + errorEvent = "Error" + recordsEvent = "Records" + progressEvent = "Progress" + statsEvent = "Stats" +) + +// contentType represents content type of event. +type contentType string + +const ( + xmlContent contentType = "text/xml" +) + +// SelectObjectContent is a implementation of http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html AWS S3 API. +func (c Client) SelectObjectContent(ctx context.Context, bucketName, objectName string, opts SelectObjectOptions) (*SelectResults, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return nil, err + } + + selectReqBytes, err := xml.Marshal(opts) + if err != nil { + return nil, err + } + + urlValues := make(url.Values) + urlValues.Set("select", "") + urlValues.Set("select-type", "2") + + // Execute POST on bucket/object. + resp, err := c.executeMethod(ctx, "POST", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + customHeader: opts.Header(), + contentMD5Base64: sumMD5Base64(selectReqBytes), + contentSHA256Hex: sum256Hex(selectReqBytes), + contentBody: bytes.NewReader(selectReqBytes), + contentLength: int64(len(selectReqBytes)), + }) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp, bucketName, "") + } + + pipeReader, pipeWriter := io.Pipe() + streamer := &SelectResults{ + resp: resp, + stats: &StatsMessage{}, + progress: &ProgressMessage{}, + pipeReader: pipeReader, + } + streamer.start(pipeWriter) + return streamer, nil +} + +// Close - closes the underlying response body and the stream reader. +func (s *SelectResults) Close() error { + defer closeResponse(s.resp) + return s.pipeReader.Close() +} + +// Read - is a reader compatible implementation for SelectObjectContent records. +func (s *SelectResults) Read(b []byte) (n int, err error) { + return s.pipeReader.Read(b) +} + +// Stats - information about a request's stats when processing is complete. +func (s *SelectResults) Stats() *StatsMessage { + return s.stats +} + +// Progress - information about the progress of a request. +func (s *SelectResults) Progress() *ProgressMessage { + return s.progress +} + +// start is the main function that decodes the large byte array into +// several events that are sent through the eventstream. +func (s *SelectResults) start(pipeWriter *io.PipeWriter) { + go func() { + for { + var prelude preludeInfo + var headers = make(http.Header) + var err error + + // Create CRC code + crc := crc32.New(crc32.IEEETable) + crcReader := io.TeeReader(s.resp.Body, crc) + + // Extract the prelude(12 bytes) into a struct to extract relevant information. + prelude, err = processPrelude(crcReader, crc) + if err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + + // Extract the headers(variable bytes) into a struct to extract relevant information + if prelude.headerLen > 0 { + if err = extractHeader(io.LimitReader(crcReader, int64(prelude.headerLen)), headers); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + } + + // Get the actual payload length so that the appropriate amount of + // bytes can be read or parsed. + payloadLen := prelude.PayloadLen() + + // Get content-type of the payload. + c := contentType(headers.Get("content-type")) + + // Get event type of the payload. + e := eventType(headers.Get("event-type")) + + // Handle all supported events. + switch e { + case endEvent: + pipeWriter.Close() + closeResponse(s.resp) + return + case errorEvent: + pipeWriter.CloseWithError(errors.New("Error Type of " + headers.Get("error-type") + " " + headers.Get("error-message"))) + closeResponse(s.resp) + return + case recordsEvent: + if _, err = io.Copy(pipeWriter, io.LimitReader(crcReader, payloadLen)); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + case progressEvent: + switch c { + case xmlContent: + if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.progress); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + default: + pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, progressEvent)) + closeResponse(s.resp) + return + } + case statsEvent: + switch c { + case xmlContent: + if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.stats); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + default: + pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, statsEvent)) + closeResponse(s.resp) + return + } + } + + // Ensures that the full message's CRC is correct and + // that the message is not corrupted + if err := checkCRC(s.resp.Body, crc.Sum32()); err != nil { + pipeWriter.CloseWithError(err) + closeResponse(s.resp) + return + } + + } + }() +} + +// PayloadLen is a function that calculates the length of the payload. +func (p preludeInfo) PayloadLen() int64 { + return int64(p.totalLen - p.headerLen - 16) +} + +// processPrelude is the function that reads the 12 bytes of the prelude and +// ensures the CRC is correct while also extracting relevant information into +// the struct, +func processPrelude(prelude io.Reader, crc hash.Hash32) (preludeInfo, error) { + var err error + var pInfo = preludeInfo{} + + // reads total length of the message (first 4 bytes) + pInfo.totalLen, err = extractUint32(prelude) + if err != nil { + return pInfo, err + } + + // reads total header length of the message (2nd 4 bytes) + pInfo.headerLen, err = extractUint32(prelude) + if err != nil { + return pInfo, err + } + + // checks that the CRC is correct (3rd 4 bytes) + preCRC := crc.Sum32() + if err := checkCRC(prelude, preCRC); err != nil { + return pInfo, err + } + + return pInfo, nil +} + +// extracts the relevant information from the Headers. +func extractHeader(body io.Reader, myHeaders http.Header) error { + for { + // extracts the first part of the header, + headerTypeName, err := extractHeaderType(body) + if err != nil { + // Since end of file, we have read all of our headers + if err == io.EOF { + break + } + return err + } + + // reads the 7 present in the header and ignores it. + extractUint8(body) + + headerValueName, err := extractHeaderValue(body) + if err != nil { + return err + } + + myHeaders.Set(headerTypeName, headerValueName) + + } + return nil +} + +// extractHeaderType extracts the first half of the header message, the header type. +func extractHeaderType(body io.Reader) (string, error) { + // extracts 2 bit integer + headerNameLen, err := extractUint8(body) + if err != nil { + return "", err + } + // extracts the string with the appropriate number of bytes + headerName, err := extractString(body, int(headerNameLen)) + if err != nil { + return "", err + } + return strings.TrimPrefix(headerName, ":"), nil +} + +// extractsHeaderValue extracts the second half of the header message, the +// header value +func extractHeaderValue(body io.Reader) (string, error) { + bodyLen, err := extractUint16(body) + if err != nil { + return "", err + } + bodyName, err := extractString(body, int(bodyLen)) + if err != nil { + return "", err + } + return bodyName, nil +} + +// extracts a string from byte array of a particular number of bytes. +func extractString(source io.Reader, lenBytes int) (string, error) { + myVal := make([]byte, lenBytes) + _, err := source.Read(myVal) + if err != nil { + return "", err + } + return string(myVal), nil +} + +// extractUint32 extracts a 4 byte integer from the byte array. +func extractUint32(r io.Reader) (uint32, error) { + buf := make([]byte, 4) + _, err := io.ReadFull(r, buf) + if err != nil { + return 0, err + } + return binary.BigEndian.Uint32(buf), nil +} + +// extractUint16 extracts a 2 byte integer from the byte array. +func extractUint16(r io.Reader) (uint16, error) { + buf := make([]byte, 2) + _, err := io.ReadFull(r, buf) + if err != nil { + return 0, err + } + return binary.BigEndian.Uint16(buf), nil +} + +// extractUint8 extracts a 1 byte integer from the byte array. +func extractUint8(r io.Reader) (uint8, error) { + buf := make([]byte, 1) + _, err := io.ReadFull(r, buf) + if err != nil { + return 0, err + } + return buf[0], nil +} + +// checkCRC ensures that the CRC matches with the one from the reader. +func checkCRC(r io.Reader, expect uint32) error { + msgCRC, err := extractUint32(r) + if err != nil { + return err + } + + if msgCRC != expect { + return fmt.Errorf("Checksum Mismatch, MessageCRC of 0x%X does not equal expected CRC of 0x%X", msgCRC, expect) + + } + return nil +} diff --git a/api-stat.go b/api-stat.go new file mode 100644 index 0000000..91e9d39 --- /dev/null +++ b/api-stat.go @@ -0,0 +1,185 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "net/http" + "strconv" + "strings" + "time" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// BucketExists verify if bucket exists and you have permission to access it. +func (c Client) BucketExists(bucketName string) (bool, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return false, err + } + + // Execute HEAD on bucketName. + resp, err := c.executeMethod(context.Background(), "HEAD", requestMetadata{ + bucketName: bucketName, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + if ToErrorResponse(err).Code == "NoSuchBucket" { + return false, nil + } + return false, err + } + if resp != nil { + resperr := httpRespToErrorResponse(resp, bucketName, "") + if ToErrorResponse(resperr).Code == "NoSuchBucket" { + return false, nil + } + if resp.StatusCode != http.StatusOK { + return false, httpRespToErrorResponse(resp, bucketName, "") + } + } + return true, nil +} + +// List of header keys to be filtered, usually +// from all S3 API http responses. +var defaultFilterKeys = []string{ + "Connection", + "Transfer-Encoding", + "Accept-Ranges", + "Date", + "Server", + "Vary", + "x-amz-bucket-region", + "x-amz-request-id", + "x-amz-id-2", + "Content-Security-Policy", + "X-Xss-Protection", + + // Add new headers to be ignored. +} + +// Extract only necessary metadata header key/values by +// filtering them out with a list of custom header keys. +func extractObjMetadata(header http.Header) http.Header { + filterKeys := append([]string{ + "ETag", + "Content-Length", + "Last-Modified", + "Content-Type", + }, defaultFilterKeys...) + return filterHeader(header, filterKeys) +} + +// StatObject verifies if object exists and you have permission to access. +func (c Client) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return ObjectInfo{}, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return ObjectInfo{}, err + } + return c.statObject(context.Background(), bucketName, objectName, opts) +} + +// Lower level API for statObject supporting pre-conditions and range headers. +func (c Client) statObject(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return ObjectInfo{}, err + } + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return ObjectInfo{}, err + } + + // Execute HEAD on objectName. + resp, err := c.executeMethod(ctx, "HEAD", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + contentSHA256Hex: emptySHA256Hex, + customHeader: opts.Header(), + }) + defer closeResponse(resp) + if err != nil { + return ObjectInfo{}, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + return ObjectInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) + } + } + + // Trim off the odd double quotes from ETag in the beginning and end. + md5sum := strings.TrimPrefix(resp.Header.Get("ETag"), "\"") + md5sum = strings.TrimSuffix(md5sum, "\"") + + // Parse content length is exists + var size int64 = -1 + contentLengthStr := resp.Header.Get("Content-Length") + if contentLengthStr != "" { + size, err = strconv.ParseInt(contentLengthStr, 10, 64) + if err != nil { + // Content-Length is not valid + return ObjectInfo{}, ErrorResponse{ + Code: "InternalError", + Message: "Content-Length is invalid. " + reportIssue, + BucketName: bucketName, + Key: objectName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), + } + } + } + + // Parse Last-Modified has http time format. + date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) + if err != nil { + return ObjectInfo{}, ErrorResponse{ + Code: "InternalError", + Message: "Last-Modified time format is invalid. " + reportIssue, + BucketName: bucketName, + Key: objectName, + RequestID: resp.Header.Get("x-amz-request-id"), + HostID: resp.Header.Get("x-amz-id-2"), + Region: resp.Header.Get("x-amz-bucket-region"), + } + } + + // Fetch content type if any present. + contentType := strings.TrimSpace(resp.Header.Get("Content-Type")) + if contentType == "" { + contentType = "application/octet-stream" + } + + // Save object metadata info. + return ObjectInfo{ + ETag: md5sum, + Key: objectName, + Size: size, + LastModified: date, + ContentType: contentType, + // Extract only the relevant header keys describing the object. + // following function filters out a list of standard set of keys + // which are not part of object metadata. + Metadata: extractObjMetadata(resp.Header), + }, nil +} diff --git a/api.go b/api.go new file mode 100644 index 0000000..79b0bc2 --- /dev/null +++ b/api.go @@ -0,0 +1,887 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha256" + "errors" + "fmt" + "hash" + "io" + "io/ioutil" + "math/rand" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "runtime" + "strings" + "sync" + "time" + + "github.com/minio/minio-go/pkg/credentials" + "github.com/minio/minio-go/pkg/s3signer" + "github.com/minio/minio-go/pkg/s3utils" +) + +// Client implements Amazon S3 compatible methods. +type Client struct { + /// Standard options. + + // Parsed endpoint url provided by the user. + endpointURL *url.URL + + // Holds various credential providers. + credsProvider *credentials.Credentials + + // Custom signerType value overrides all credentials. + overrideSignerType credentials.SignatureType + + // User supplied. + appInfo struct { + appName string + appVersion string + } + + // Indicate whether we are using https or not + secure bool + + // Needs allocation. + httpClient *http.Client + bucketLocCache *bucketLocationCache + + // Advanced functionality. + isTraceEnabled bool + traceOutput io.Writer + + // S3 specific accelerated endpoint. + s3AccelerateEndpoint string + + // Region endpoint + region string + + // Random seed. + random *rand.Rand + + // lookup indicates type of url lookup supported by server. If not specified, + // default to Auto. + lookup BucketLookupType +} + +// Options for New method +type Options struct { + Creds *credentials.Credentials + Secure bool + Region string + BucketLookup BucketLookupType + // Add future fields here +} + +// Global constants. +const ( + libraryName = "minio-go" + libraryVersion = "v6.0.11" +) + +// User Agent should always following the below style. +// Please open an issue to discuss any new changes here. +// +// Minio (OS; ARCH) LIB/VER APP/VER +const ( + libraryUserAgentPrefix = "Minio (" + runtime.GOOS + "; " + runtime.GOARCH + ") " + libraryUserAgent = libraryUserAgentPrefix + libraryName + "/" + libraryVersion +) + +// BucketLookupType is type of url lookup supported by server. +type BucketLookupType int + +// Different types of url lookup supported by the server.Initialized to BucketLookupAuto +const ( + BucketLookupAuto BucketLookupType = iota + BucketLookupDNS + BucketLookupPath +) + +// NewV2 - instantiate minio client with Amazon S3 signature version +// '2' compatibility. +func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { + creds := credentials.NewStaticV2(accessKeyID, secretAccessKey, "") + clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto) + if err != nil { + return nil, err + } + clnt.overrideSignerType = credentials.SignatureV2 + return clnt, nil +} + +// NewV4 - instantiate minio client with Amazon S3 signature version +// '4' compatibility. +func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { + creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") + clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto) + if err != nil { + return nil, err + } + clnt.overrideSignerType = credentials.SignatureV4 + return clnt, nil +} + +// New - instantiate minio client, adds automatic verification of signature. +func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { + creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") + clnt, err := privateNew(endpoint, creds, secure, "", BucketLookupAuto) + if err != nil { + return nil, err + } + // Google cloud storage should be set to signature V2, force it if not. + if s3utils.IsGoogleEndpoint(*clnt.endpointURL) { + clnt.overrideSignerType = credentials.SignatureV2 + } + // If Amazon S3 set to signature v4. + if s3utils.IsAmazonEndpoint(*clnt.endpointURL) { + clnt.overrideSignerType = credentials.SignatureV4 + } + return clnt, nil +} + +// NewWithCredentials - instantiate minio client with credentials provider +// for retrieving credentials from various credentials provider such as +// IAM, File, Env etc. +func NewWithCredentials(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) { + return privateNew(endpoint, creds, secure, region, BucketLookupAuto) +} + +// NewWithRegion - instantiate minio client, with region configured. Unlike New(), +// NewWithRegion avoids bucket-location lookup operations and it is slightly faster. +// Use this function when if your application deals with single region. +func NewWithRegion(endpoint, accessKeyID, secretAccessKey string, secure bool, region string) (*Client, error) { + creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") + return privateNew(endpoint, creds, secure, region, BucketLookupAuto) +} + +// NewWithOptions - instantiate minio client with options +func NewWithOptions(endpoint string, opts *Options) (*Client, error) { + return privateNew(endpoint, opts.Creds, opts.Secure, opts.Region, opts.BucketLookup) +} + +// lockedRandSource provides protected rand source, implements rand.Source interface. +type lockedRandSource struct { + lk sync.Mutex + src rand.Source +} + +// Int63 returns a non-negative pseudo-random 63-bit integer as an int64. +func (r *lockedRandSource) Int63() (n int64) { + r.lk.Lock() + n = r.src.Int63() + r.lk.Unlock() + return +} + +// Seed uses the provided seed value to initialize the generator to a +// deterministic state. +func (r *lockedRandSource) Seed(seed int64) { + r.lk.Lock() + r.src.Seed(seed) + r.lk.Unlock() +} + +// Redirect requests by re signing the request. +func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error { + if len(via) >= 5 { + return errors.New("stopped after 5 redirects") + } + if len(via) == 0 { + return nil + } + lastRequest := via[len(via)-1] + var reAuth bool + for attr, val := range lastRequest.Header { + // if hosts do not match do not copy Authorization header + if attr == "Authorization" && req.Host != lastRequest.Host { + reAuth = true + continue + } + if _, ok := req.Header[attr]; !ok { + req.Header[attr] = val + } + } + + *c.endpointURL = *req.URL + + value, err := c.credsProvider.Get() + if err != nil { + return err + } + var ( + signerType = value.SignerType + accessKeyID = value.AccessKeyID + secretAccessKey = value.SecretAccessKey + sessionToken = value.SessionToken + region = c.region + ) + + // Custom signer set then override the behavior. + if c.overrideSignerType != credentials.SignatureDefault { + signerType = c.overrideSignerType + } + + // If signerType returned by credentials helper is anonymous, + // then do not sign regardless of signerType override. + if value.SignerType == credentials.SignatureAnonymous { + signerType = credentials.SignatureAnonymous + } + + if reAuth { + // Check if there is no region override, if not get it from the URL if possible. + if region == "" { + region = s3utils.GetRegionFromURL(*c.endpointURL) + } + switch { + case signerType.IsV2(): + return errors.New("signature V2 cannot support redirection") + case signerType.IsV4(): + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region)) + } + } + return nil +} + +func privateNew(endpoint string, creds *credentials.Credentials, secure bool, region string, lookup BucketLookupType) (*Client, error) { + // construct endpoint. + endpointURL, err := getEndpointURL(endpoint, secure) + if err != nil { + return nil, err + } + + // instantiate new Client. + clnt := new(Client) + + // Save the credentials. + clnt.credsProvider = creds + + // Remember whether we are using https or not + clnt.secure = secure + + // Save endpoint URL, user agent for future uses. + clnt.endpointURL = endpointURL + + // Instantiate http client and bucket location cache. + clnt.httpClient = &http.Client{ + Transport: DefaultTransport, + CheckRedirect: clnt.redirectHeaders, + } + + // Sets custom region, if region is empty bucket location cache is used automatically. + if region == "" { + region = s3utils.GetRegionFromURL(*clnt.endpointURL) + } + clnt.region = region + + // Instantiate bucket location cache. + clnt.bucketLocCache = newBucketLocationCache() + + // Introduce a new locked random seed. + clnt.random = rand.New(&lockedRandSource{src: rand.NewSource(time.Now().UTC().UnixNano())}) + + // Sets bucket lookup style, whether server accepts DNS or Path lookup. Default is Auto - determined + // by the SDK. When Auto is specified, DNS lookup is used for Amazon/Google cloud endpoints and Path for all other endpoints. + clnt.lookup = lookup + // Return. + return clnt, nil +} + +// SetAppInfo - add application details to user agent. +func (c *Client) SetAppInfo(appName string, appVersion string) { + // if app name and version not set, we do not set a new user agent. + if appName != "" && appVersion != "" { + c.appInfo = struct { + appName string + appVersion string + }{} + c.appInfo.appName = appName + c.appInfo.appVersion = appVersion + } +} + +// SetCustomTransport - set new custom transport. +func (c *Client) SetCustomTransport(customHTTPTransport http.RoundTripper) { + // Set this to override default transport + // ``http.DefaultTransport``. + // + // This transport is usually needed for debugging OR to add your + // own custom TLS certificates on the client transport, for custom + // CA's and certs which are not part of standard certificate + // authority follow this example :- + // + // tr := &http.Transport{ + // TLSClientConfig: &tls.Config{RootCAs: pool}, + // DisableCompression: true, + // } + // api.SetCustomTransport(tr) + // + if c.httpClient != nil { + c.httpClient.Transport = customHTTPTransport + } +} + +// TraceOn - enable HTTP tracing. +func (c *Client) TraceOn(outputStream io.Writer) { + // if outputStream is nil then default to os.Stdout. + if outputStream == nil { + outputStream = os.Stdout + } + // Sets a new output stream. + c.traceOutput = outputStream + + // Enable tracing. + c.isTraceEnabled = true +} + +// TraceOff - disable HTTP tracing. +func (c *Client) TraceOff() { + // Disable tracing. + c.isTraceEnabled = false +} + +// SetS3TransferAccelerate - turns s3 accelerated endpoint on or off for all your +// requests. This feature is only specific to S3 for all other endpoints this +// function does nothing. To read further details on s3 transfer acceleration +// please vist - +// http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html +func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) { + if s3utils.IsAmazonEndpoint(*c.endpointURL) { + c.s3AccelerateEndpoint = accelerateEndpoint + } +} + +// Hash materials provides relevant initialized hash algo writers +// based on the expected signature type. +// +// - For signature v4 request if the connection is insecure compute only sha256. +// - For signature v4 request if the connection is secure compute only md5. +// - For anonymous request compute md5. +func (c *Client) hashMaterials() (hashAlgos map[string]hash.Hash, hashSums map[string][]byte) { + hashSums = make(map[string][]byte) + hashAlgos = make(map[string]hash.Hash) + if c.overrideSignerType.IsV4() { + if c.secure { + hashAlgos["md5"] = md5.New() + } else { + hashAlgos["sha256"] = sha256.New() + } + } else { + if c.overrideSignerType.IsAnonymous() { + hashAlgos["md5"] = md5.New() + } + } + return hashAlgos, hashSums +} + +// requestMetadata - is container for all the values to make a request. +type requestMetadata struct { + // If set newRequest presigns the URL. + presignURL bool + + // User supplied. + bucketName string + objectName string + queryValues url.Values + customHeader http.Header + expires int64 + + // Generated by our internal code. + bucketLocation string + contentBody io.Reader + contentLength int64 + contentMD5Base64 string // carries base64 encoded md5sum + contentSHA256Hex string // carries hex encoded sha256sum +} + +// dumpHTTP - dump HTTP request and response. +func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { + // Starts http dump. + _, err := fmt.Fprintln(c.traceOutput, "---------START-HTTP---------") + if err != nil { + return err + } + + // Filter out Signature field from Authorization header. + origAuth := req.Header.Get("Authorization") + if origAuth != "" { + req.Header.Set("Authorization", redactSignature(origAuth)) + } + + // Only display request header. + reqTrace, err := httputil.DumpRequestOut(req, false) + if err != nil { + return err + } + + // Write request to trace output. + _, err = fmt.Fprint(c.traceOutput, string(reqTrace)) + if err != nil { + return err + } + + // Only display response header. + var respTrace []byte + + // For errors we make sure to dump response body as well. + if resp.StatusCode != http.StatusOK && + resp.StatusCode != http.StatusPartialContent && + resp.StatusCode != http.StatusNoContent { + respTrace, err = httputil.DumpResponse(resp, true) + if err != nil { + return err + } + } else { + respTrace, err = httputil.DumpResponse(resp, false) + if err != nil { + return err + } + } + + // Write response to trace output. + _, err = fmt.Fprint(c.traceOutput, strings.TrimSuffix(string(respTrace), "\r\n")) + if err != nil { + return err + } + + // Ends the http dump. + _, err = fmt.Fprintln(c.traceOutput, "---------END-HTTP---------") + if err != nil { + return err + } + + // Returns success. + return nil +} + +// do - execute http request. +func (c Client) do(req *http.Request) (*http.Response, error) { + resp, err := c.httpClient.Do(req) + if err != nil { + // Handle this specifically for now until future Golang versions fix this issue properly. + if urlErr, ok := err.(*url.Error); ok { + if strings.Contains(urlErr.Err.Error(), "EOF") { + return nil, &url.Error{ + Op: urlErr.Op, + URL: urlErr.URL, + Err: errors.New("Connection closed by foreign host " + urlErr.URL + ". Retry again."), + } + } + } + return nil, err + } + + // Response cannot be non-nil, report error if thats the case. + if resp == nil { + msg := "Response is empty. " + reportIssue + return nil, ErrInvalidArgument(msg) + } + + // If trace is enabled, dump http request and response. + if c.isTraceEnabled { + err = c.dumpHTTP(req, resp) + if err != nil { + return nil, err + } + } + + return resp, nil +} + +// List of success status. +var successStatus = []int{ + http.StatusOK, + http.StatusNoContent, + http.StatusPartialContent, +} + +// executeMethod - instantiates a given method, and retries the +// request upon any error up to maxRetries attempts in a binomially +// delayed manner using a standard back off algorithm. +func (c Client) executeMethod(ctx context.Context, method string, metadata requestMetadata) (res *http.Response, err error) { + var isRetryable bool // Indicates if request can be retried. + var bodySeeker io.Seeker // Extracted seeker from io.Reader. + var reqRetry = MaxRetry // Indicates how many times we can retry the request + + if metadata.contentBody != nil { + // Check if body is seekable then it is retryable. + bodySeeker, isRetryable = metadata.contentBody.(io.Seeker) + switch bodySeeker { + case os.Stdin, os.Stdout, os.Stderr: + isRetryable = false + } + // Retry only when reader is seekable + if !isRetryable { + reqRetry = 1 + } + + // Figure out if the body can be closed - if yes + // we will definitely close it upon the function + // return. + bodyCloser, ok := metadata.contentBody.(io.Closer) + if ok { + defer bodyCloser.Close() + } + } + + // Create a done channel to control 'newRetryTimer' go routine. + doneCh := make(chan struct{}, 1) + + // Indicate to our routine to exit cleanly upon return. + defer close(doneCh) + + // Blank indentifier is kept here on purpose since 'range' without + // blank identifiers is only supported since go1.4 + // https://golang.org/doc/go1.4#forrange. + for range c.newRetryTimer(reqRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter, doneCh) { + // Retry executes the following function body if request has an + // error until maxRetries have been exhausted, retry attempts are + // performed after waiting for a given period of time in a + // binomial fashion. + if isRetryable { + // Seek back to beginning for each attempt. + if _, err = bodySeeker.Seek(0, 0); err != nil { + // If seek failed, no need to retry. + return nil, err + } + } + + // Instantiate a new request. + var req *http.Request + req, err = c.newRequest(method, metadata) + if err != nil { + errResponse := ToErrorResponse(err) + if isS3CodeRetryable(errResponse.Code) { + continue // Retry. + } + return nil, err + } + + // Add context to request + req = req.WithContext(ctx) + + // Initiate the request. + res, err = c.do(req) + if err != nil { + // For supported http requests errors verify. + if isHTTPReqErrorRetryable(err) { + continue // Retry. + } + // For other errors, return here no need to retry. + return nil, err + } + + // For any known successful http status, return quickly. + for _, httpStatus := range successStatus { + if httpStatus == res.StatusCode { + return res, nil + } + } + + // Read the body to be saved later. + errBodyBytes, err := ioutil.ReadAll(res.Body) + // res.Body should be closed + closeResponse(res) + if err != nil { + return nil, err + } + + // Save the body. + errBodySeeker := bytes.NewReader(errBodyBytes) + res.Body = ioutil.NopCloser(errBodySeeker) + + // For errors verify if its retryable otherwise fail quickly. + errResponse := ToErrorResponse(httpRespToErrorResponse(res, metadata.bucketName, metadata.objectName)) + + // Save the body back again. + errBodySeeker.Seek(0, 0) // Seek back to starting point. + res.Body = ioutil.NopCloser(errBodySeeker) + + // Bucket region if set in error response and the error + // code dictates invalid region, we can retry the request + // with the new region. + // + // Additionally we should only retry if bucketLocation and custom + // region is empty. + if metadata.bucketLocation == "" && c.region == "" { + if errResponse.Code == "AuthorizationHeaderMalformed" || errResponse.Code == "InvalidRegion" { + if metadata.bucketName != "" && errResponse.Region != "" { + // Gather Cached location only if bucketName is present. + if _, cachedLocationError := c.bucketLocCache.Get(metadata.bucketName); cachedLocationError != false { + c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) + continue // Retry. + } + } + } + } + + // Verify if error response code is retryable. + if isS3CodeRetryable(errResponse.Code) { + continue // Retry. + } + + // Verify if http status code is retryable. + if isHTTPStatusRetryable(res.StatusCode) { + continue // Retry. + } + + // For all other cases break out of the retry loop. + break + } + return res, err +} + +// newRequest - instantiate a new HTTP request for a given method. +func (c Client) newRequest(method string, metadata requestMetadata) (req *http.Request, err error) { + // If no method is supplied default to 'POST'. + if method == "" { + method = "POST" + } + + location := metadata.bucketLocation + if location == "" { + if metadata.bucketName != "" { + // Gather location only if bucketName is present. + location, err = c.getBucketLocation(metadata.bucketName) + if err != nil { + if ToErrorResponse(err).Code != "AccessDenied" { + return nil, err + } + } + // Upon AccessDenied error on fetching bucket location, default + // to possible locations based on endpoint URL. This can usually + // happen when GetBucketLocation() is disabled using IAM policies. + } + if location == "" { + location = getDefaultLocation(*c.endpointURL, c.region) + } + } + + // Look if target url supports virtual host. + isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, metadata.bucketName) + + // Construct a new target URL. + targetURL, err := c.makeTargetURL(metadata.bucketName, metadata.objectName, location, isVirtualHost, metadata.queryValues) + if err != nil { + return nil, err + } + + // Initialize a new HTTP request for the method. + req, err = http.NewRequest(method, targetURL.String(), nil) + if err != nil { + return nil, err + } + + // Get credentials from the configured credentials provider. + value, err := c.credsProvider.Get() + if err != nil { + return nil, err + } + + var ( + signerType = value.SignerType + accessKeyID = value.AccessKeyID + secretAccessKey = value.SecretAccessKey + sessionToken = value.SessionToken + ) + + // Custom signer set then override the behavior. + if c.overrideSignerType != credentials.SignatureDefault { + signerType = c.overrideSignerType + } + + // If signerType returned by credentials helper is anonymous, + // then do not sign regardless of signerType override. + if value.SignerType == credentials.SignatureAnonymous { + signerType = credentials.SignatureAnonymous + } + + // Generate presign url if needed, return right here. + if metadata.expires != 0 && metadata.presignURL { + if signerType.IsAnonymous() { + return nil, ErrInvalidArgument("Presigned URLs cannot be generated with anonymous credentials.") + } + if signerType.IsV2() { + // Presign URL with signature v2. + req = s3signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost) + } else if signerType.IsV4() { + // Presign URL with signature v4. + req = s3signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires) + } + return req, nil + } + + // Set 'User-Agent' header for the request. + c.setUserAgent(req) + + // Set all headers. + for k, v := range metadata.customHeader { + req.Header.Set(k, v[0]) + } + + // Go net/http notoriously closes the request body. + // - The request Body, if non-nil, will be closed by the underlying Transport, even on errors. + // This can cause underlying *os.File seekers to fail, avoid that + // by making sure to wrap the closer as a nop. + if metadata.contentLength == 0 { + req.Body = nil + } else { + req.Body = ioutil.NopCloser(metadata.contentBody) + } + + // Set incoming content-length. + req.ContentLength = metadata.contentLength + if req.ContentLength <= -1 { + // For unknown content length, we upload using transfer-encoding: chunked. + req.TransferEncoding = []string{"chunked"} + } + + // set md5Sum for content protection. + if len(metadata.contentMD5Base64) > 0 { + req.Header.Set("Content-Md5", metadata.contentMD5Base64) + } + + // For anonymous requests just return. + if signerType.IsAnonymous() { + return req, nil + } + + switch { + case signerType.IsV2(): + // Add signature version '2' authorization header. + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) + case metadata.objectName != "" && method == "PUT" && metadata.customHeader.Get("X-Amz-Copy-Source") == "" && !c.secure: + // Streaming signature is used by default for a PUT object request. Additionally we also + // look if the initialized client is secure, if yes then we don't need to perform + // streaming signature. + req = s3signer.StreamingSignV4(req, accessKeyID, + secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC()) + default: + // Set sha256 sum for signature calculation only with signature version '4'. + shaHeader := unsignedPayload + if metadata.contentSHA256Hex != "" { + shaHeader = metadata.contentSHA256Hex + } + req.Header.Set("X-Amz-Content-Sha256", shaHeader) + + // Add signature version '4' authorization header. + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, location) + } + + // Return request. + return req, nil +} + +// set User agent. +func (c Client) setUserAgent(req *http.Request) { + req.Header.Set("User-Agent", libraryUserAgent) + if c.appInfo.appName != "" && c.appInfo.appVersion != "" { + req.Header.Set("User-Agent", libraryUserAgent+" "+c.appInfo.appName+"/"+c.appInfo.appVersion) + } +} + +// makeTargetURL make a new target url. +func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error) { + host := c.endpointURL.Host + // For Amazon S3 endpoint, try to fetch location based endpoint. + if s3utils.IsAmazonEndpoint(*c.endpointURL) { + if c.s3AccelerateEndpoint != "" && bucketName != "" { + // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html + // Disable transfer acceleration for non-compliant bucket names. + if strings.Contains(bucketName, ".") { + return nil, ErrTransferAccelerationBucket(bucketName) + } + // If transfer acceleration is requested set new host. + // For more details about enabling transfer acceleration read here. + // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html + host = c.s3AccelerateEndpoint + } else { + // Do not change the host if the endpoint URL is a FIPS S3 endpoint. + if !s3utils.IsAmazonFIPSEndpoint(*c.endpointURL) { + // Fetch new host based on the bucket location. + host = getS3Endpoint(bucketLocation) + } + } + } + + // Save scheme. + scheme := c.endpointURL.Scheme + + // Strip port 80 and 443 so we won't send these ports in Host header. + // The reason is that browsers and curl automatically remove :80 and :443 + // with the generated presigned urls, then a signature mismatch error. + if h, p, err := net.SplitHostPort(host); err == nil { + if scheme == "http" && p == "80" || scheme == "https" && p == "443" { + host = h + } + } + + urlStr := scheme + "://" + host + "/" + // Make URL only if bucketName is available, otherwise use the + // endpoint URL. + if bucketName != "" { + // If endpoint supports virtual host style use that always. + // Currently only S3 and Google Cloud Storage would support + // virtual host style. + if isVirtualHostStyle { + urlStr = scheme + "://" + bucketName + "." + host + "/" + if objectName != "" { + urlStr = urlStr + s3utils.EncodePath(objectName) + } + } else { + // If not fall back to using path style. + urlStr = urlStr + bucketName + "/" + if objectName != "" { + urlStr = urlStr + s3utils.EncodePath(objectName) + } + } + } + + // If there are any query values, add them to the end. + if len(queryValues) > 0 { + urlStr = urlStr + "?" + s3utils.QueryEncode(queryValues) + } + + return url.Parse(urlStr) +} + +// returns true if virtual hosted style requests are to be used. +func (c *Client) isVirtualHostStyleRequest(url url.URL, bucketName string) bool { + if bucketName == "" { + return false + } + + if c.lookup == BucketLookupDNS { + return true + } + if c.lookup == BucketLookupPath { + return false + } + + // default to virtual only for Amazon/Google storage. In all other cases use + // path style requests + return s3utils.IsVirtualHostSupported(url, bucketName) +} diff --git a/api_unit_test.go b/api_unit_test.go new file mode 100644 index 0000000..d7822ab --- /dev/null +++ b/api_unit_test.go @@ -0,0 +1,210 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "net/http" + "net/url" + "testing" + + "github.com/minio/minio-go/pkg/credentials" + "github.com/minio/minio-go/pkg/policy" +) + +type customReader struct{} + +func (c *customReader) Read(p []byte) (n int, err error) { + return 0, nil +} + +func (c *customReader) Size() (n int64) { + return 10 +} + +// Tests valid hosts for location. +func TestValidBucketLocation(t *testing.T) { + s3Hosts := []struct { + bucketLocation string + endpoint string + }{ + {"us-east-1", "s3.amazonaws.com"}, + {"unknown", "s3.amazonaws.com"}, + {"ap-southeast-1", "s3-ap-southeast-1.amazonaws.com"}, + } + for _, s3Host := range s3Hosts { + endpoint := getS3Endpoint(s3Host.bucketLocation) + if endpoint != s3Host.endpoint { + t.Fatal("Error: invalid bucket location", endpoint) + } + } +} + +// Tests error response structure. +func TestErrorResponse(t *testing.T) { + var err error + err = ErrorResponse{ + Code: "Testing", + } + errResp := ToErrorResponse(err) + if errResp.Code != "Testing" { + t.Fatal("Type conversion failed, we have an empty struct.") + } + + // Test http response decoding. + var httpResponse *http.Response + // Set empty variables + httpResponse = nil + var bucketName, objectName string + + // Should fail with invalid argument. + err = httpRespToErrorResponse(httpResponse, bucketName, objectName) + errResp = ToErrorResponse(err) + if errResp.Code != "InvalidArgument" { + t.Fatal("Empty response input should return invalid argument.") + } +} + +// Tests signature type. +func TestSignatureType(t *testing.T) { + clnt := Client{} + if !clnt.overrideSignerType.IsV4() { + t.Fatal("Error") + } + clnt.overrideSignerType = credentials.SignatureV2 + if !clnt.overrideSignerType.IsV2() { + t.Fatal("Error") + } + if clnt.overrideSignerType.IsV4() { + t.Fatal("Error") + } + clnt.overrideSignerType = credentials.SignatureV4 + if !clnt.overrideSignerType.IsV4() { + t.Fatal("Error") + } +} + +// Tests bucket policy types. +func TestBucketPolicyTypes(t *testing.T) { + want := map[string]bool{ + "none": true, + "readonly": true, + "writeonly": true, + "readwrite": true, + "invalid": false, + } + for bucketPolicy, ok := range want { + if policy.BucketPolicy(bucketPolicy).IsValidBucketPolicy() != ok { + t.Fatal("Error") + } + } +} + +// Tests optimal part size. +func TestPartSize(t *testing.T) { + _, _, _, err := optimalPartInfo(5000000000000000000) + if err == nil { + t.Fatal("Error: should fail") + } + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(5497558138880) + if err != nil { + t.Fatal("Error: ", err) + } + if totalPartsCount != 9103 { + t.Fatalf("Error: expecting total parts count of 9987: got %v instead", totalPartsCount) + } + if partSize != 603979776 { + t.Fatalf("Error: expecting part size of 550502400: got %v instead", partSize) + } + if lastPartSize != 134217728 { + t.Fatalf("Error: expecting last part size of 241172480: got %v instead", lastPartSize) + } + _, partSize, _, err = optimalPartInfo(5000000000) + if err != nil { + t.Fatal("Error:", err) + } + if partSize != minPartSize { + t.Fatalf("Error: expecting part size of %v: got %v instead", minPartSize, partSize) + } + totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(-1) + if err != nil { + t.Fatal("Error:", err) + } + if totalPartsCount != 9103 { + t.Fatalf("Error: expecting total parts count of 9987: got %v instead", totalPartsCount) + } + if partSize != 603979776 { + t.Fatalf("Error: expecting part size of 550502400: got %v instead", partSize) + } + if lastPartSize != 134217728 { + t.Fatalf("Error: expecting last part size of 241172480: got %v instead", lastPartSize) + } +} + +// TestMakeTargetURL - testing makeTargetURL() +func TestMakeTargetURL(t *testing.T) { + testCases := []struct { + addr string + secure bool + bucketName string + objectName string + bucketLocation string + queryValues map[string][]string + expectedURL url.URL + expectedErr error + }{ + // Test 1 + {"localhost:9000", false, "", "", "", nil, url.URL{Host: "localhost:9000", Scheme: "http", Path: "/"}, nil}, + // Test 2 + {"localhost", true, "", "", "", nil, url.URL{Host: "localhost", Scheme: "https", Path: "/"}, nil}, + // Test 3 + {"localhost:9000", true, "mybucket", "", "", nil, url.URL{Host: "localhost:9000", Scheme: "https", Path: "/mybucket/"}, nil}, + // Test 4, testing against google storage API + {"storage.googleapis.com", true, "mybucket", "", "", nil, url.URL{Host: "mybucket.storage.googleapis.com", Scheme: "https", Path: "/"}, nil}, + // Test 5, testing against AWS S3 API + {"s3.amazonaws.com", true, "mybucket", "myobject", "", nil, url.URL{Host: "mybucket.s3.amazonaws.com", Scheme: "https", Path: "/myobject"}, nil}, + // Test 6 + {"localhost:9000", false, "mybucket", "myobject", "", nil, url.URL{Host: "localhost:9000", Scheme: "http", Path: "/mybucket/myobject"}, nil}, + // Test 7, testing with query + {"localhost:9000", false, "mybucket", "myobject", "", map[string][]string{"param": {"val"}}, url.URL{Host: "localhost:9000", Scheme: "http", Path: "/mybucket/myobject", RawQuery: "param=val"}, nil}, + // Test 8, testing with port 80 + {"localhost:80", false, "mybucket", "myobject", "", nil, url.URL{Host: "localhost", Scheme: "http", Path: "/mybucket/myobject"}, nil}, + // Test 9, testing with port 443 + {"localhost:443", true, "mybucket", "myobject", "", nil, url.URL{Host: "localhost", Scheme: "https", Path: "/mybucket/myobject"}, nil}, + } + + for i, testCase := range testCases { + // Initialize a Minio client + c, _ := New(testCase.addr, "foo", "bar", testCase.secure) + isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, testCase.bucketName) + u, err := c.makeTargetURL(testCase.bucketName, testCase.objectName, testCase.bucketLocation, isVirtualHost, testCase.queryValues) + // Check the returned error + if testCase.expectedErr == nil && err != nil { + t.Fatalf("Test %d: Should succeed but failed with err = %v", i+1, err) + } + if testCase.expectedErr != nil && err == nil { + t.Fatalf("Test %d: Should fail but succeeded", i+1) + } + if err == nil { + // Check if the returned url is equal to what we expect + if u.String() != testCase.expectedURL.String() { + t.Fatalf("Test %d: Mismatched target url: expected = `%v`, found = `%v`", + i+1, testCase.expectedURL.String(), u.String()) + } + } + } +} diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..48ea6e7 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,39 @@ +# version format +version: "{build}" + +# Operating system (build VM template) +os: Windows Server 2012 R2 + +clone_folder: c:\gopath\src\github.com\minio\minio-go + +# environment variables +environment: + GOPATH: c:\gopath + GO15VENDOREXPERIMENT: 1 + +# scripts that run after cloning repository +install: + - set PATH=%GOPATH%\bin;c:\go\bin;%PATH% + - go version + - go env + - go get -u golang.org/x/lint/golint + - go get -u github.com/remyoudompheng/go-misc/deadcode + - go get -u github.com/gordonklaus/ineffassign + - go get -u golang.org/x/crypto/argon2 + - go get -t ./... + +# to run your custom scripts instead of automatic MSBuild +build_script: + - go vet ./... + - gofmt -s -l . + - golint -set_exit_status github.com/minio/minio-go... + - deadcode + - ineffassign . + - go test -short -v + - go test -short -race -v + +# to disable automatic tests +test: off + +# to disable deployment +deploy: off diff --git a/bucket-cache.go b/bucket-cache.go new file mode 100644 index 0000000..cac7ad7 --- /dev/null +++ b/bucket-cache.go @@ -0,0 +1,221 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "net/http" + "net/url" + "path" + "sync" + + "github.com/minio/minio-go/pkg/credentials" + "github.com/minio/minio-go/pkg/s3signer" + "github.com/minio/minio-go/pkg/s3utils" +) + +// bucketLocationCache - Provides simple mechanism to hold bucket +// locations in memory. +type bucketLocationCache struct { + // mutex is used for handling the concurrent + // read/write requests for cache. + sync.RWMutex + + // items holds the cached bucket locations. + items map[string]string +} + +// newBucketLocationCache - Provides a new bucket location cache to be +// used internally with the client object. +func newBucketLocationCache() *bucketLocationCache { + return &bucketLocationCache{ + items: make(map[string]string), + } +} + +// Get - Returns a value of a given key if it exists. +func (r *bucketLocationCache) Get(bucketName string) (location string, ok bool) { + r.RLock() + defer r.RUnlock() + location, ok = r.items[bucketName] + return +} + +// Set - Will persist a value into cache. +func (r *bucketLocationCache) Set(bucketName string, location string) { + r.Lock() + defer r.Unlock() + r.items[bucketName] = location +} + +// Delete - Deletes a bucket name from cache. +func (r *bucketLocationCache) Delete(bucketName string) { + r.Lock() + defer r.Unlock() + delete(r.items, bucketName) +} + +// GetBucketLocation - get location for the bucket name from location cache, if not +// fetch freshly by making a new request. +func (c Client) GetBucketLocation(bucketName string) (string, error) { + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return "", err + } + return c.getBucketLocation(bucketName) +} + +// getBucketLocation - Get location for the bucketName from location map cache, if not +// fetch freshly by making a new request. +func (c Client) getBucketLocation(bucketName string) (string, error) { + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return "", err + } + + // Region set then no need to fetch bucket location. + if c.region != "" { + return c.region, nil + } + + if location, ok := c.bucketLocCache.Get(bucketName); ok { + return location, nil + } + + // Initialize a new request. + req, err := c.getBucketLocationRequest(bucketName) + if err != nil { + return "", err + } + + // Initiate the request. + resp, err := c.do(req) + defer closeResponse(resp) + if err != nil { + return "", err + } + location, err := processBucketLocationResponse(resp, bucketName) + if err != nil { + return "", err + } + c.bucketLocCache.Set(bucketName, location) + return location, nil +} + +// processes the getBucketLocation http response from the server. +func processBucketLocationResponse(resp *http.Response, bucketName string) (bucketLocation string, err error) { + if resp != nil { + if resp.StatusCode != http.StatusOK { + err = httpRespToErrorResponse(resp, bucketName, "") + errResp := ToErrorResponse(err) + // For access denied error, it could be an anonymous + // request. Move forward and let the top level callers + // succeed if possible based on their policy. + if errResp.Code == "AccessDenied" { + return "us-east-1", nil + } + return "", err + } + } + + // Extract location. + var locationConstraint string + err = xmlDecoder(resp.Body, &locationConstraint) + if err != nil { + return "", err + } + + location := locationConstraint + // Location is empty will be 'us-east-1'. + if location == "" { + location = "us-east-1" + } + + // Location can be 'EU' convert it to meaningful 'eu-west-1'. + if location == "EU" { + location = "eu-west-1" + } + + // Save the location into cache. + + // Return. + return location, nil +} + +// getBucketLocationRequest - Wrapper creates a new getBucketLocation request. +func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, error) { + // Set location query. + urlValues := make(url.Values) + urlValues.Set("location", "") + + // Set get bucket location always as path style. + targetURL := c.endpointURL + targetURL.Path = path.Join(bucketName, "") + "/" + targetURL.RawQuery = urlValues.Encode() + + // Get a new HTTP request for the method. + req, err := http.NewRequest("GET", targetURL.String(), nil) + if err != nil { + return nil, err + } + + // Set UserAgent for the request. + c.setUserAgent(req) + + // Get credentials from the configured credentials provider. + value, err := c.credsProvider.Get() + if err != nil { + return nil, err + } + + var ( + signerType = value.SignerType + accessKeyID = value.AccessKeyID + secretAccessKey = value.SecretAccessKey + sessionToken = value.SessionToken + ) + + // Custom signer set then override the behavior. + if c.overrideSignerType != credentials.SignatureDefault { + signerType = c.overrideSignerType + } + + // If signerType returned by credentials helper is anonymous, + // then do not sign regardless of signerType override. + if value.SignerType == credentials.SignatureAnonymous { + signerType = credentials.SignatureAnonymous + } + + if signerType.IsAnonymous() { + return req, nil + } + + if signerType.IsV2() { + // Get Bucket Location calls should be always path style + isVirtualHost := false + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) + return req, nil + } + + // Set sha256 sum for signature calculation only with signature version '4'. + contentSha256 := emptySHA256Hex + if c.secure { + contentSha256 = unsignedPayload + } + + req.Header.Set("X-Amz-Content-Sha256", contentSha256) + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") + return req, nil +} diff --git a/bucket-cache_test.go b/bucket-cache_test.go new file mode 100644 index 0000000..507b408 --- /dev/null +++ b/bucket-cache_test.go @@ -0,0 +1,351 @@ +/* + * Copyright + * 2015, 2016, 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "encoding/xml" + "io/ioutil" + "net/http" + "net/url" + "path" + "reflect" + "testing" + + "github.com/minio/minio-go/pkg/credentials" + "github.com/minio/minio-go/pkg/s3signer" +) + +// Test validates `newBucketLocationCache`. +func TestNewBucketLocationCache(t *testing.T) { + expectedBucketLocationcache := &bucketLocationCache{ + items: make(map[string]string), + } + actualBucketLocationCache := newBucketLocationCache() + + if !reflect.DeepEqual(actualBucketLocationCache, expectedBucketLocationcache) { + t.Errorf("Unexpected return value") + } +} + +// Tests validate bucketLocationCache operations. +func TestBucketLocationCacheOps(t *testing.T) { + testBucketLocationCache := newBucketLocationCache() + expectedBucketName := "minio-bucket" + expectedLocation := "us-east-1" + testBucketLocationCache.Set(expectedBucketName, expectedLocation) + actualLocation, ok := testBucketLocationCache.Get(expectedBucketName) + if !ok { + t.Errorf("Bucket location cache not set") + } + if expectedLocation != actualLocation { + t.Errorf("Bucket location cache not set to expected value") + } + testBucketLocationCache.Delete(expectedBucketName) + _, ok = testBucketLocationCache.Get(expectedBucketName) + if ok { + t.Errorf("Bucket location cache not deleted as expected") + } +} + +// Tests validate http request generation for 'getBucketLocation'. +func TestGetBucketLocationRequest(t *testing.T) { + // Generates expected http request for getBucketLocation. + // Used for asserting with the actual request generated. + createExpectedRequest := func(c *Client, bucketName string, req *http.Request) (*http.Request, error) { + // Set location query. + urlValues := make(url.Values) + urlValues.Set("location", "") + + // Set get bucket location always as path style. + targetURL := c.endpointURL + targetURL.Path = path.Join(bucketName, "") + "/" + targetURL.RawQuery = urlValues.Encode() + + // Get a new HTTP request for the method. + var err error + req, err = http.NewRequest("GET", targetURL.String(), nil) + if err != nil { + return nil, err + } + + // Set UserAgent for the request. + c.setUserAgent(req) + + // Get credentials from the configured credentials provider. + value, err := c.credsProvider.Get() + if err != nil { + return nil, err + } + + var ( + signerType = value.SignerType + accessKeyID = value.AccessKeyID + secretAccessKey = value.SecretAccessKey + sessionToken = value.SessionToken + ) + + // Custom signer set then override the behavior. + if c.overrideSignerType != credentials.SignatureDefault { + signerType = c.overrideSignerType + } + + // If signerType returned by credentials helper is anonymous, + // then do not sign regardless of signerType override. + if value.SignerType == credentials.SignatureAnonymous { + signerType = credentials.SignatureAnonymous + } + + // Set sha256 sum for signature calculation only + // with signature version '4'. + switch { + case signerType.IsV4(): + contentSha256 := emptySHA256Hex + if c.secure { + contentSha256 = unsignedPayload + } + req.Header.Set("X-Amz-Content-Sha256", contentSha256) + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") + case signerType.IsV2(): + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey, false) + } + + return req, nil + + } + // Info for 'Client' creation. + // Will be used as arguments for 'NewClient'. + type infoForClient struct { + endPoint string + accessKey string + secretKey string + enableInsecure bool + } + // dataset for 'NewClient' call. + info := []infoForClient{ + // endpoint localhost. + // both access-key and secret-key are empty. + {"localhost:9000", "", "", false}, + // both access-key are secret-key exists. + {"localhost:9000", "my-access-key", "my-secret-key", false}, + // one of acess-key and secret-key are empty. + {"localhost:9000", "", "my-secret-key", false}, + + // endpoint amazon s3. + {"s3.amazonaws.com", "", "", false}, + {"s3.amazonaws.com", "my-access-key", "my-secret-key", false}, + {"s3.amazonaws.com", "my-acess-key", "", false}, + + // endpoint google cloud storage. + {"storage.googleapis.com", "", "", false}, + {"storage.googleapis.com", "my-access-key", "my-secret-key", false}, + {"storage.googleapis.com", "", "my-secret-key", false}, + + // endpoint custom domain running Minio server. + {"play.minio.io", "", "", false}, + {"play.minio.io", "my-access-key", "my-secret-key", false}, + {"play.minio.io", "my-acess-key", "", false}, + } + testCases := []struct { + bucketName string + // data for new client creation. + info infoForClient + // error in the output. + err error + // flag indicating whether tests should pass. + shouldPass bool + }{ + // Client is constructed using the info struct. + // case with empty location. + {"my-bucket", info[0], nil, true}, + // case with location set to standard 'us-east-1'. + {"my-bucket", info[0], nil, true}, + // case with location set to a value different from 'us-east-1'. + {"my-bucket", info[0], nil, true}, + + {"my-bucket", info[1], nil, true}, + {"my-bucket", info[1], nil, true}, + {"my-bucket", info[1], nil, true}, + + {"my-bucket", info[2], nil, true}, + {"my-bucket", info[2], nil, true}, + {"my-bucket", info[2], nil, true}, + + {"my-bucket", info[3], nil, true}, + {"my-bucket", info[3], nil, true}, + {"my-bucket", info[3], nil, true}, + + {"my-bucket", info[4], nil, true}, + {"my-bucket", info[4], nil, true}, + {"my-bucket", info[4], nil, true}, + + {"my-bucket", info[5], nil, true}, + {"my-bucket", info[5], nil, true}, + {"my-bucket", info[5], nil, true}, + + {"my-bucket", info[6], nil, true}, + {"my-bucket", info[6], nil, true}, + {"my-bucket", info[6], nil, true}, + + {"my-bucket", info[7], nil, true}, + {"my-bucket", info[7], nil, true}, + {"my-bucket", info[7], nil, true}, + + {"my-bucket", info[8], nil, true}, + {"my-bucket", info[8], nil, true}, + {"my-bucket", info[8], nil, true}, + + {"my-bucket", info[9], nil, true}, + {"my-bucket", info[9], nil, true}, + {"my-bucket", info[9], nil, true}, + + {"my-bucket", info[10], nil, true}, + {"my-bucket", info[10], nil, true}, + {"my-bucket", info[10], nil, true}, + + {"my-bucket", info[11], nil, true}, + {"my-bucket", info[11], nil, true}, + {"my-bucket", info[11], nil, true}, + } + for i, testCase := range testCases { + // cannot create a newclient with empty endPoint value. + // validates and creates a new client only if the endPoint value is not empty. + client := &Client{} + var err error + if testCase.info.endPoint != "" { + + client, err = New(testCase.info.endPoint, testCase.info.accessKey, testCase.info.secretKey, testCase.info.enableInsecure) + if err != nil { + t.Fatalf("Test %d: Failed to create new Client: %s", i+1, err.Error()) + } + } + + actualReq, err := client.getBucketLocationRequest(testCase.bucketName) + if err != nil && testCase.shouldPass { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) + } + if err == nil && !testCase.shouldPass { + t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if err.Error() != testCase.err.Error() { + t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) + } + } + + // Test passes as expected, but the output values are verified for correctness here. + if err == nil && testCase.shouldPass { + expectedReq := &http.Request{} + expectedReq, err = createExpectedRequest(client, testCase.bucketName, expectedReq) + if err != nil { + t.Fatalf("Test %d: Expected request Creation failed", i+1) + } + if expectedReq.Method != actualReq.Method { + t.Errorf("Test %d: The expected Request method doesn't match with the actual one", i+1) + } + if expectedReq.URL.String() != actualReq.URL.String() { + t.Errorf("Test %d: Expected the request URL to be '%s', but instead found '%s'", i+1, expectedReq.URL.String(), actualReq.URL.String()) + } + if expectedReq.ContentLength != actualReq.ContentLength { + t.Errorf("Test %d: Expected the request body Content-Length to be '%d', but found '%d' instead", i+1, expectedReq.ContentLength, actualReq.ContentLength) + } + + if expectedReq.Header.Get("X-Amz-Content-Sha256") != actualReq.Header.Get("X-Amz-Content-Sha256") { + t.Errorf("Test %d: 'X-Amz-Content-Sha256' header of the expected request doesn't match with that of the actual request", i+1) + } + if expectedReq.Header.Get("User-Agent") != actualReq.Header.Get("User-Agent") { + t.Errorf("Test %d: Expected 'User-Agent' header to be \"%s\",but found \"%s\" instead", i+1, expectedReq.Header.Get("User-Agent"), actualReq.Header.Get("User-Agent")) + } + } + } +} + +// generates http response with bucket location set in the body. +func generateLocationResponse(resp *http.Response, bodyContent []byte) (*http.Response, error) { + resp.StatusCode = http.StatusOK + resp.Body = ioutil.NopCloser(bytes.NewBuffer(bodyContent)) + return resp, nil +} + +// Tests the processing of GetPolicy response from server. +func TestProcessBucketLocationResponse(t *testing.T) { + // LocationResponse - format for location response. + type LocationResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"` + Location string `xml:",chardata"` + } + + APIErrors := []APIError{ + { + Code: "AccessDenied", + Description: "Access Denied", + HTTPStatusCode: http.StatusUnauthorized, + }, + } + testCases := []struct { + bucketName string + inputLocation string + isAPIError bool + apiErr APIError + // expected results. + expectedResult string + err error + // flag indicating whether tests should pass. + shouldPass bool + }{ + {"my-bucket", "", true, APIErrors[0], "us-east-1", nil, true}, + {"my-bucket", "", false, APIError{}, "us-east-1", nil, true}, + {"my-bucket", "EU", false, APIError{}, "eu-west-1", nil, true}, + {"my-bucket", "eu-central-1", false, APIError{}, "eu-central-1", nil, true}, + {"my-bucket", "us-east-1", false, APIError{}, "us-east-1", nil, true}, + } + + for i, testCase := range testCases { + inputResponse := &http.Response{} + var err error + if testCase.isAPIError { + inputResponse = generateErrorResponse(inputResponse, testCase.apiErr, testCase.bucketName) + } else { + inputResponse, err = generateLocationResponse(inputResponse, encodeResponse(LocationResponse{ + Location: testCase.inputLocation, + })) + if err != nil { + t.Fatalf("Test %d: Creation of valid response failed", i+1) + } + } + actualResult, err := processBucketLocationResponse(inputResponse, "my-bucket") + if err != nil && testCase.shouldPass { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) + } + if err == nil && !testCase.shouldPass { + t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if err.Error() != testCase.err.Error() { + t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) + } + } + if err == nil && testCase.shouldPass { + if !reflect.DeepEqual(testCase.expectedResult, actualResult) { + t.Errorf("Test %d: The expected BucketPolicy doesn't match the actual BucketPolicy", i+1) + } + } + } +} diff --git a/bucket-notification.go b/bucket-notification.go new file mode 100644 index 0000000..ea303dd --- /dev/null +++ b/bucket-notification.go @@ -0,0 +1,273 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "encoding/xml" + + "github.com/minio/minio-go/pkg/set" +) + +// NotificationEventType is a S3 notification event associated to the bucket notification configuration +type NotificationEventType string + +// The role of all event types are described in : +// http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html#notification-how-to-event-types-and-destinations +const ( + ObjectCreatedAll NotificationEventType = "s3:ObjectCreated:*" + ObjectCreatedPut = "s3:ObjectCreated:Put" + ObjectCreatedPost = "s3:ObjectCreated:Post" + ObjectCreatedCopy = "s3:ObjectCreated:Copy" + ObjectCreatedCompleteMultipartUpload = "s3:ObjectCreated:CompleteMultipartUpload" + ObjectAccessedGet = "s3:ObjectAccessed:Get" + ObjectAccessedHead = "s3:ObjectAccessed:Head" + ObjectAccessedAll = "s3:ObjectAccessed:*" + ObjectRemovedAll = "s3:ObjectRemoved:*" + ObjectRemovedDelete = "s3:ObjectRemoved:Delete" + ObjectRemovedDeleteMarkerCreated = "s3:ObjectRemoved:DeleteMarkerCreated" + ObjectReducedRedundancyLostObject = "s3:ReducedRedundancyLostObject" +) + +// FilterRule - child of S3Key, a tag in the notification xml which +// carries suffix/prefix filters +type FilterRule struct { + Name string `xml:"Name"` + Value string `xml:"Value"` +} + +// S3Key - child of Filter, a tag in the notification xml which +// carries suffix/prefix filters +type S3Key struct { + FilterRules []FilterRule `xml:"FilterRule,omitempty"` +} + +// Filter - a tag in the notification xml structure which carries +// suffix/prefix filters +type Filter struct { + S3Key S3Key `xml:"S3Key,omitempty"` +} + +// Arn - holds ARN information that will be sent to the web service, +// ARN desciption can be found in http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html +type Arn struct { + Partition string + Service string + Region string + AccountID string + Resource string +} + +// NewArn creates new ARN based on the given partition, service, region, account id and resource +func NewArn(partition, service, region, accountID, resource string) Arn { + return Arn{Partition: partition, + Service: service, + Region: region, + AccountID: accountID, + Resource: resource} +} + +// Return the string format of the ARN +func (arn Arn) String() string { + return "arn:" + arn.Partition + ":" + arn.Service + ":" + arn.Region + ":" + arn.AccountID + ":" + arn.Resource +} + +// NotificationConfig - represents one single notification configuration +// such as topic, queue or lambda configuration. +type NotificationConfig struct { + ID string `xml:"Id,omitempty"` + Arn Arn `xml:"-"` + Events []NotificationEventType `xml:"Event"` + Filter *Filter `xml:"Filter,omitempty"` +} + +// NewNotificationConfig creates one notification config and sets the given ARN +func NewNotificationConfig(arn Arn) NotificationConfig { + return NotificationConfig{Arn: arn, Filter: &Filter{}} +} + +// AddEvents adds one event to the current notification config +func (t *NotificationConfig) AddEvents(events ...NotificationEventType) { + t.Events = append(t.Events, events...) +} + +// AddFilterSuffix sets the suffix configuration to the current notification config +func (t *NotificationConfig) AddFilterSuffix(suffix string) { + if t.Filter == nil { + t.Filter = &Filter{} + } + newFilterRule := FilterRule{Name: "suffix", Value: suffix} + // Replace any suffix rule if existing and add to the list otherwise + for index := range t.Filter.S3Key.FilterRules { + if t.Filter.S3Key.FilterRules[index].Name == "suffix" { + t.Filter.S3Key.FilterRules[index] = newFilterRule + return + } + } + t.Filter.S3Key.FilterRules = append(t.Filter.S3Key.FilterRules, newFilterRule) +} + +// AddFilterPrefix sets the prefix configuration to the current notification config +func (t *NotificationConfig) AddFilterPrefix(prefix string) { + if t.Filter == nil { + t.Filter = &Filter{} + } + newFilterRule := FilterRule{Name: "prefix", Value: prefix} + // Replace any prefix rule if existing and add to the list otherwise + for index := range t.Filter.S3Key.FilterRules { + if t.Filter.S3Key.FilterRules[index].Name == "prefix" { + t.Filter.S3Key.FilterRules[index] = newFilterRule + return + } + } + t.Filter.S3Key.FilterRules = append(t.Filter.S3Key.FilterRules, newFilterRule) +} + +// TopicConfig carries one single topic notification configuration +type TopicConfig struct { + NotificationConfig + Topic string `xml:"Topic"` +} + +// QueueConfig carries one single queue notification configuration +type QueueConfig struct { + NotificationConfig + Queue string `xml:"Queue"` +} + +// LambdaConfig carries one single cloudfunction notification configuration +type LambdaConfig struct { + NotificationConfig + Lambda string `xml:"CloudFunction"` +} + +// BucketNotification - the struct that represents the whole XML to be sent to the web service +type BucketNotification struct { + XMLName xml.Name `xml:"NotificationConfiguration"` + LambdaConfigs []LambdaConfig `xml:"CloudFunctionConfiguration"` + TopicConfigs []TopicConfig `xml:"TopicConfiguration"` + QueueConfigs []QueueConfig `xml:"QueueConfiguration"` +} + +// AddTopic adds a given topic config to the general bucket notification config +func (b *BucketNotification) AddTopic(topicConfig NotificationConfig) bool { + newTopicConfig := TopicConfig{NotificationConfig: topicConfig, Topic: topicConfig.Arn.String()} + for _, n := range b.TopicConfigs { + // If new config matches existing one + if n.Topic == newTopicConfig.Arn.String() && newTopicConfig.Filter == n.Filter { + + existingConfig := set.NewStringSet() + for _, v := range n.Events { + existingConfig.Add(string(v)) + } + + newConfig := set.NewStringSet() + for _, v := range topicConfig.Events { + newConfig.Add(string(v)) + } + + if !newConfig.Intersection(existingConfig).IsEmpty() { + return false + } + } + } + b.TopicConfigs = append(b.TopicConfigs, newTopicConfig) + return true +} + +// AddQueue adds a given queue config to the general bucket notification config +func (b *BucketNotification) AddQueue(queueConfig NotificationConfig) bool { + newQueueConfig := QueueConfig{NotificationConfig: queueConfig, Queue: queueConfig.Arn.String()} + for _, n := range b.QueueConfigs { + if n.Queue == newQueueConfig.Arn.String() && newQueueConfig.Filter == n.Filter { + + existingConfig := set.NewStringSet() + for _, v := range n.Events { + existingConfig.Add(string(v)) + } + + newConfig := set.NewStringSet() + for _, v := range queueConfig.Events { + newConfig.Add(string(v)) + } + + if !newConfig.Intersection(existingConfig).IsEmpty() { + return false + } + } + } + b.QueueConfigs = append(b.QueueConfigs, newQueueConfig) + return true +} + +// AddLambda adds a given lambda config to the general bucket notification config +func (b *BucketNotification) AddLambda(lambdaConfig NotificationConfig) bool { + newLambdaConfig := LambdaConfig{NotificationConfig: lambdaConfig, Lambda: lambdaConfig.Arn.String()} + for _, n := range b.LambdaConfigs { + if n.Lambda == newLambdaConfig.Arn.String() && newLambdaConfig.Filter == n.Filter { + + existingConfig := set.NewStringSet() + for _, v := range n.Events { + existingConfig.Add(string(v)) + } + + newConfig := set.NewStringSet() + for _, v := range lambdaConfig.Events { + newConfig.Add(string(v)) + } + + if !newConfig.Intersection(existingConfig).IsEmpty() { + return false + } + } + } + b.LambdaConfigs = append(b.LambdaConfigs, newLambdaConfig) + return true +} + +// RemoveTopicByArn removes all topic configurations that match the exact specified ARN +func (b *BucketNotification) RemoveTopicByArn(arn Arn) { + var topics []TopicConfig + for _, topic := range b.TopicConfigs { + if topic.Topic != arn.String() { + topics = append(topics, topic) + } + } + b.TopicConfigs = topics +} + +// RemoveQueueByArn removes all queue configurations that match the exact specified ARN +func (b *BucketNotification) RemoveQueueByArn(arn Arn) { + var queues []QueueConfig + for _, queue := range b.QueueConfigs { + if queue.Queue != arn.String() { + queues = append(queues, queue) + } + } + b.QueueConfigs = queues +} + +// RemoveLambdaByArn removes all lambda configurations that match the exact specified ARN +func (b *BucketNotification) RemoveLambdaByArn(arn Arn) { + var lambdas []LambdaConfig + for _, lambda := range b.LambdaConfigs { + if lambda.Lambda != arn.String() { + lambdas = append(lambdas, lambda) + } + } + b.LambdaConfigs = lambdas +} diff --git a/constants.go b/constants.go new file mode 100644 index 0000000..7377423 --- /dev/null +++ b/constants.go @@ -0,0 +1,62 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +/// Multipart upload defaults. + +// absMinPartSize - absolute minimum part size (5 MiB) below which +// a part in a multipart upload may not be uploaded. +const absMinPartSize = 1024 * 1024 * 5 + +// minPartSize - minimum part size 64MiB per object after which +// putObject behaves internally as multipart. +const minPartSize = 1024 * 1024 * 64 + +// maxPartsCount - maximum number of parts for a single multipart session. +const maxPartsCount = 10000 + +// maxPartSize - maximum part size 5GiB for a single multipart upload +// operation. +const maxPartSize = 1024 * 1024 * 1024 * 5 + +// maxSinglePutObjectSize - maximum size 5GiB of object per PUT +// operation. +const maxSinglePutObjectSize = 1024 * 1024 * 1024 * 5 + +// maxMultipartPutObjectSize - maximum size 5TiB of object for +// Multipart operation. +const maxMultipartPutObjectSize = 1024 * 1024 * 1024 * 1024 * 5 + +// unsignedPayload - value to be set to X-Amz-Content-Sha256 header when +// we don't want to sign the request payload +const unsignedPayload = "UNSIGNED-PAYLOAD" + +// Total number of parallel workers used for multipart operation. +const totalWorkers = 4 + +// Signature related constants. +const ( + signV4Algorithm = "AWS4-HMAC-SHA256" + iso8601DateFormat = "20060102T150405Z" +) + +// Storage class header constant. +const amzStorageClass = "X-Amz-Storage-Class" + +// Website redirect location header constant +const amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location" diff --git a/core.go b/core.go new file mode 100644 index 0000000..4d51363 --- /dev/null +++ b/core.go @@ -0,0 +1,153 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "context" + "io" + "strings" + + "github.com/minio/minio-go/pkg/encrypt" +) + +// Core - Inherits Client and adds new methods to expose the low level S3 APIs. +type Core struct { + *Client +} + +// NewCore - Returns new initialized a Core client, this CoreClient should be +// only used under special conditions such as need to access lower primitives +// and being able to use them to write your own wrappers. +func NewCore(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Core, error) { + var s3Client Core + client, err := NewV4(endpoint, accessKeyID, secretAccessKey, secure) + if err != nil { + return nil, err + } + s3Client.Client = client + return &s3Client, nil +} + +// ListObjects - List all the objects at a prefix, optionally with marker and delimiter +// you can further filter the results. +func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListBucketResult, err error) { + return c.listObjectsQuery(bucket, prefix, marker, delimiter, maxKeys) +} + +// ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses +// continuationToken instead of marker to support iteration over the results. +func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { + return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys, startAfter) +} + +// CopyObject - copies an object from source object to destination object on server side. +func (c Core) CopyObject(sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { + return c.copyObjectDo(context.Background(), sourceBucket, sourceObject, destBucket, destObject, metadata) +} + +// CopyObjectPart - creates a part in a multipart upload by copying (a +// part of) an existing object. +func (c Core) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, + partID int, startOffset, length int64, metadata map[string]string) (p CompletePart, err error) { + + return c.copyObjectPartDo(context.Background(), srcBucket, srcObject, destBucket, destObject, uploadID, + partID, startOffset, length, metadata) +} + +// PutObject - Upload object. Uploads using single PUT call. +func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) { + opts := PutObjectOptions{} + m := make(map[string]string) + for k, v := range metadata { + if strings.ToLower(k) == "content-encoding" { + opts.ContentEncoding = v + } else if strings.ToLower(k) == "content-disposition" { + opts.ContentDisposition = v + } else if strings.ToLower(k) == "content-language" { + opts.ContentLanguage = v + } else if strings.ToLower(k) == "content-type" { + opts.ContentType = v + } else if strings.ToLower(k) == "cache-control" { + opts.CacheControl = v + } else if strings.ToLower(k) == strings.ToLower(amzWebsiteRedirectLocation) { + opts.WebsiteRedirectLocation = v + } else { + m[k] = metadata[k] + } + } + opts.UserMetadata = m + opts.ServerSideEncryption = sse + return c.putObjectDo(context.Background(), bucket, object, data, md5Base64, sha256Hex, size, opts) +} + +// NewMultipartUpload - Initiates new multipart upload and returns the new uploadID. +func (c Core) NewMultipartUpload(bucket, object string, opts PutObjectOptions) (uploadID string, err error) { + result, err := c.initiateMultipartUpload(context.Background(), bucket, object, opts) + return result.UploadID, err +} + +// ListMultipartUploads - List incomplete uploads. +func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartUploadsResult, err error) { + return c.listMultipartUploadsQuery(bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads) +} + +// PutObjectPart - Upload an object part. +func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { + return c.uploadPart(context.Background(), bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse) +} + +// ListObjectParts - List uploaded parts of an incomplete upload.x +func (c Core) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListObjectPartsResult, err error) { + return c.listObjectPartsQuery(bucket, object, uploadID, partNumberMarker, maxParts) +} + +// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. +func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) (string, error) { + res, err := c.completeMultipartUpload(context.Background(), bucket, object, uploadID, completeMultipartUpload{ + Parts: parts, + }) + return res.ETag, err +} + +// AbortMultipartUpload - Abort an incomplete upload. +func (c Core) AbortMultipartUpload(bucket, object, uploadID string) error { + return c.abortMultipartUpload(context.Background(), bucket, object, uploadID) +} + +// GetBucketPolicy - fetches bucket access policy for a given bucket. +func (c Core) GetBucketPolicy(bucket string) (string, error) { + return c.getBucketPolicy(bucket) +} + +// PutBucketPolicy - applies a new bucket access policy for a given bucket. +func (c Core) PutBucketPolicy(bucket, bucketPolicy string) error { + return c.putBucketPolicy(bucket, bucketPolicy) +} + +// GetObject is a lower level API implemented to support reading +// partial objects and also downloading objects with special conditions +// matching etag, modtime etc. +func (c Core) GetObject(bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, error) { + return c.getObject(context.Background(), bucketName, objectName, opts) +} + +// StatObject is a lower level API implemented to support special +// conditions matching etag, modtime on a request. +func (c Core) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { + return c.statObject(context.Background(), bucketName, objectName, opts) +} diff --git a/core_test.go b/core_test.go new file mode 100644 index 0000000..5afdd18 --- /dev/null +++ b/core_test.go @@ -0,0 +1,770 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "io" + "log" + "os" + "testing" + "time" + + "math/rand" +) + +const ( + serverEndpoint = "SERVER_ENDPOINT" + accessKey = "ACCESS_KEY" + secretKey = "SECRET_KEY" + enableSecurity = "ENABLE_HTTPS" +) + +// Minimum part size +const MinPartSize = 1024 * 1024 * 64 +const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + return prefix + string(b[0:30-len(prefix)]) +} + +// Tests for Core GetObject() function. +func TestGetObjectCore(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio core client object. + c, err := NewCore( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("3"), rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.Client.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), PutObjectOptions{ + ContentType: "binary/octet-stream", + }) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + offset := int64(2048) + + // read directly + buf1 := make([]byte, 512) + buf2 := make([]byte, 512) + buf3 := make([]byte, n) + buf4 := make([]byte, 1) + + opts := GetObjectOptions{} + opts.SetRange(offset, offset+int64(len(buf1))-1) + reader, objectInfo, err := c.GetObject(bucketName, objectName, opts) + if err != nil { + t.Fatal(err) + } + m, err := io.ReadFull(reader, buf1) + reader.Close() + if err != nil { + t.Fatal(err) + } + + if objectInfo.Size != int64(m) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) + } + if !bytes.Equal(buf1, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two GetObject from same offset.") + } + offset += 512 + + opts.SetRange(offset, offset+int64(len(buf2))-1) + reader, objectInfo, err = c.GetObject(bucketName, objectName, opts) + if err != nil { + t.Fatal(err) + } + + m, err = io.ReadFull(reader, buf2) + reader.Close() + if err != nil { + t.Fatal(err) + } + + if objectInfo.Size != int64(m) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two GetObject from same offset.") + } + + opts.SetRange(0, int64(len(buf3))) + reader, objectInfo, err = c.GetObject(bucketName, objectName, opts) + if err != nil { + t.Fatal(err) + } + + m, err = io.ReadFull(reader, buf3) + if err != nil { + reader.Close() + t.Fatal(err) + } + reader.Close() + + if objectInfo.Size != int64(m) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) + } + if !bytes.Equal(buf3, buf) { + t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") + } + + opts = GetObjectOptions{} + opts.SetMatchETag("etag") + _, _, err = c.GetObject(bucketName, objectName, opts) + if err == nil { + t.Fatal("Unexpected GetObject should fail with mismatching etags") + } + if errResp := ToErrorResponse(err); errResp.Code != "PreconditionFailed" { + t.Fatalf("Expected \"PreconditionFailed\" as code, got %s instead", errResp.Code) + } + + opts = GetObjectOptions{} + opts.SetMatchETagExcept("etag") + reader, objectInfo, err = c.GetObject(bucketName, objectName, opts) + if err != nil { + t.Fatal(err) + } + + m, err = io.ReadFull(reader, buf3) + reader.Close() + if err != nil { + t.Fatal(err) + } + + if objectInfo.Size != int64(m) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) + } + if !bytes.Equal(buf3, buf) { + t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") + } + + opts = GetObjectOptions{} + opts.SetRange(0, 0) + reader, objectInfo, err = c.GetObject(bucketName, objectName, opts) + if err != nil { + t.Fatal(err) + } + + m, err = io.ReadFull(reader, buf4) + reader.Close() + if err != nil { + t.Fatal(err) + } + + if objectInfo.Size != int64(m) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests GetObject to return Content-Encoding properly set +// and overrides any auto decoding. +func TestGetObjectContentEncoding(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio core client object. + c, err := NewCore( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("3"), rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.Client.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), PutObjectOptions{ + ContentEncoding: "gzip", + }) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + rwc, objInfo, err := c.GetObject(bucketName, objectName, GetObjectOptions{}) + if err != nil { + t.Fatalf("Error: %v", err) + } + rwc.Close() + if objInfo.Size <= 0 { + t.Fatalf("Unexpected size of the object %v, expected %v", objInfo.Size, n) + } + value, ok := objInfo.Metadata["Content-Encoding"] + if !ok { + t.Fatalf("Expected Content-Encoding metadata to be set.") + } + if value[0] != "gzip" { + t.Fatalf("Unexpected content-encoding found, want gzip, got %v", value) + } +} + +// Tests get bucket policy core API. +func TestGetBucketPolicy(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewCore( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Verify if bucket exits and you have access. + var exists bool + exists, err = c.BucketExists(bucketName) + if err != nil { + t.Fatal("Error:", err, bucketName) + } + if !exists { + t.Fatal("Error: could not find ", bucketName) + } + + // Asserting the default bucket policy. + bucketPolicy, err := c.GetBucketPolicy(bucketName) + if err != nil { + errResp := ToErrorResponse(err) + if errResp.Code != "NoSuchBucketPolicy" { + t.Error("Error:", err, bucketName) + } + } + if bucketPolicy != "" { + t.Errorf("Bucket policy expected %#v, got %#v", "", bucketPolicy) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests Core CopyObject API implementation. +func TestCoreCopyObject(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewCore( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + buf := bytes.Repeat([]byte("a"), 32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + "Content-Type": "binary/octet-stream", + }, nil) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if objInfo.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size) + } + + destBucketName := bucketName + destObjectName := objectName + "-dest" + + cobjInfo, err := c.CopyObject(bucketName, objectName, destBucketName, destObjectName, map[string]string{ + "X-Amz-Metadata-Directive": "REPLACE", + "Content-Type": "application/javascript", + }) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName, destBucketName, destObjectName) + } + if cobjInfo.ETag != objInfo.ETag { + t.Fatalf("Error: expected etag to be same as source object %s, but found different etag :%s", objInfo.ETag, cobjInfo.ETag) + } + + // Attempt to read from destBucketName and object name. + r, err := c.Client.GetObject(destBucketName, destObjectName, GetObjectOptions{}) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + if st.ContentType != "application/javascript" { + t.Fatalf("Error: Content types don't match, expected: application/javascript, found: %+v\n", st.ContentType) + } + + if st.ETag != objInfo.ETag { + t.Fatalf("Error: expected etag to be same as source object %s, but found different etag :%s", objInfo.ETag, st.ETag) + } + + if err := r.Close(); err != nil { + t.Fatal("Error:", err) + } + + if err := r.Close(); err == nil { + t.Fatal("Error: object is already closed, should return error") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveObject(destBucketName, destObjectName) + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + + // Do not need to remove destBucketName its same as bucketName. +} + +// Test Core CopyObjectPart implementation +func TestCoreCopyObjectPart(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewCore( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Make a buffer with 5MB of data + buf := bytes.Repeat([]byte("abcde"), 1024*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + "Content-Type": "binary/octet-stream", + }, nil) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if objInfo.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size) + } + + destBucketName := bucketName + destObjectName := objectName + "-dest" + + uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, PutObjectOptions{}) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + // Content of the destination object will be two copies of + // `objectName` concatenated, followed by first byte of + // `objectName`. + + // First of three parts + fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, nil) + if err != nil { + t.Fatal("Error:", err, destBucketName, destObjectName) + } + + // Second of three parts + sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, nil) + if err != nil { + t.Fatal("Error:", err, destBucketName, destObjectName) + } + + // Last of three parts + lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, nil) + if err != nil { + t.Fatal("Error:", err, destBucketName, destObjectName) + } + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []CompletePart{fstPart, sndPart, lstPart}) + if err != nil { + t.Fatal("Error:", err, destBucketName, destObjectName) + } + + // Stat the object and check its length matches + objInfo, err = c.StatObject(destBucketName, destObjectName, StatObjectOptions{}) + if err != nil { + t.Fatal("Error:", err, destBucketName, destObjectName) + } + + if objInfo.Size != (5*1024*1024)*2+1 { + t.Fatal("Destination object has incorrect size!") + } + + // Now we read the data back + getOpts := GetObjectOptions{} + getOpts.SetRange(0, 5*1024*1024-1) + r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + t.Fatal("Error:", err, destBucketName, destObjectName) + } + getBuf := make([]byte, 5*1024*1024) + _, err = io.ReadFull(r, getBuf) + if err != nil { + t.Fatal("Error:", err, destBucketName, destObjectName) + } + if !bytes.Equal(getBuf, buf) { + t.Fatal("Got unexpected data in first 5MB") + } + + getOpts.SetRange(5*1024*1024, 0) + r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + t.Fatal("Error:", err, destBucketName, destObjectName) + } + getBuf = make([]byte, 5*1024*1024+1) + _, err = io.ReadFull(r, getBuf) + if err != nil { + t.Fatal("Error:", err, destBucketName, destObjectName) + } + if !bytes.Equal(getBuf[:5*1024*1024], buf) { + t.Fatal("Got unexpected data in second 5MB") + } + if getBuf[5*1024*1024] != buf[0] { + t.Fatal("Got unexpected data in last byte of copied object!") + } + + if err := c.RemoveObject(destBucketName, destObjectName); err != nil { + t.Fatal("Error: ", err) + } + + if err := c.RemoveObject(bucketName, objectName); err != nil { + t.Fatal("Error: ", err) + } + + if err := c.RemoveBucket(bucketName); err != nil { + t.Fatal("Error: ", err) + } + + // Do not need to remove destBucketName its same as bucketName. +} + +// Test Core PutObject. +func TestCorePutObject(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewCore( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity)), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + buf := bytes.Repeat([]byte("a"), 32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + // Object content type + objectContentType := "binary/octet-stream" + metadata := make(map[string]string) + metadata["Content-Type"] = objectContentType + + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "1B2M2Y8AsgTpgAmY7PhCfg==", "", metadata, nil) + if err == nil { + t.Fatal("Error expected: error, got: nil(success)") + } + + objInfo, err = c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", metadata, nil) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if objInfo.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size) + } + + // Read the data back + r, err := c.Client.GetObject(bucketName, objectName, GetObjectOptions{}) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + if st.ContentType != objectContentType { + t.Fatalf("Error: Content types don't match, expected: %+v, found: %+v\n", objectContentType, st.ContentType) + } + + if err := r.Close(); err != nil { + t.Fatal("Error:", err) + } + + if err := r.Close(); err == nil { + t.Fatal("Error: object is already closed, should return error") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +func TestCoreGetObjectMetadata(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + core, err := NewCore( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableSecurity))) + if err != nil { + log.Fatalln(err) + } + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = core.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + metadata := map[string]string{ + "X-Amz-Meta-Key-1": "Val-1", + } + + _, err = core.PutObject(bucketName, "my-objectname", + bytes.NewReader([]byte("hello")), 5, "", "", metadata, nil) + if err != nil { + log.Fatalln(err) + } + + reader, objInfo, err := core.GetObject(bucketName, "my-objectname", GetObjectOptions{}) + if err != nil { + log.Fatalln(err) + } + defer reader.Close() + + if objInfo.Metadata.Get("X-Amz-Meta-Key-1") != "Val-1" { + log.Fatalln("Expected metadata to be available but wasn't") + } +} diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..5778216 --- /dev/null +++ b/docs/API.md @@ -0,0 +1,1630 @@ +# Minio Go Client API Reference [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) + +## Initialize Minio Client object. + +## Minio + +```go +package main + +import ( + "fmt" + + "github.com/minio/minio-go" +) + +func main() { + // Use a secure connection. + ssl := true + + // Initialize minio client object. + minioClient, err := minio.New("play.minio.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) + if err != nil { + fmt.Println(err) + return + } +} +``` + +## AWS S3 + +```go +package main + +import ( + "fmt" + + "github.com/minio/minio-go" +) + +func main() { + // Use a secure connection. + ssl := true + + // Initialize minio client object. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ssl) + if err != nil { + fmt.Println(err) + return + } +} +``` + +| Bucket operations | Object operations | Encrypted Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | +| :--- | :--- | :--- | :--- | :--- | :--- | +| [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`GetObject`](#GetObject) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | +| [`ListBuckets`](#ListBuckets) | [`PutObject`](#PutObject) | [`PutObject`](#PutObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | +| [`BucketExists`](#BucketExists) | [`CopyObject`](#CopyObject) | [`CopyObject`](#CopyObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | +| [`RemoveBucket`](#RemoveBucket) | [`StatObject`](#StatObject) | [`StatObject`](#StatObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | +| [`ListObjects`](#ListObjects) | [`RemoveObject`](#RemoveObject) | | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | +| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveObjects`](#RemoveObjects) | | | [`ListenBucketNotification`](#ListenBucketNotification) | | +| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | +| | [`FPutObject`](#FPutObject) | [`FPutObject`](#FPutObject) | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | +| | [`FGetObject`](#FGetObject) | [`FGetObject`](#FGetObject) | | | | +| | [`ComposeObject`](#ComposeObject) | [`ComposeObject`](#ComposeObject) | | | | +| | [`NewSourceInfo`](#NewSourceInfo) | [`NewSourceInfo`](#NewSourceInfo) | | | | +| | [`NewDestinationInfo`](#NewDestinationInfo) | [`NewDestinationInfo`](#NewDestinationInfo) | | | | +| | [`PutObjectWithContext`](#PutObjectWithContext) | [`PutObjectWithContext`](#PutObjectWithContext) | | | +| | [`GetObjectWithContext`](#GetObjectWithContext) | [`GetObjectWithContext`](#GetObjectWithContext) | | | +| | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | | +| | [`FGetObjectWithContext`](#FGetObjectWithContext) | [`FGetObjectWithContext`](#FGetObjectWithContext) | | | +| | [`RemoveObjectsWithContext`](#RemoveObjectsWithContext) | | | | +| | [`SelectObjectContent`](#SelectObjectContent) | | +## 1. Constructor + + +### New(endpoint, accessKeyID, secretAccessKey string, ssl bool) (*Client, error) +Initializes a new client object. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`endpoint` | _string_ |S3 compatible object storage endpoint | +|`accessKeyID` |_string_ |Access key for the object storage | +|`secretAccessKey` | _string_ |Secret key for the object storage | +|`ssl` | _bool_ | If 'true' API requests will be secure (HTTPS), and insecure (HTTP) otherwise | + +### NewWithRegion(endpoint, accessKeyID, secretAccessKey string, ssl bool, region string) (*Client, error) +Initializes minio client, with region configured. Unlike New(), NewWithRegion avoids bucket-location lookup operations and it is slightly faster. Use this function when your application deals with a single region. + +### NewWithOptions(endpoint string, options *Options) (*Client, error) +Initializes minio client with options configured. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`endpoint` | _string_ |S3 compatible object storage endpoint | +|`opts` |_minio.Options_ | Options for constructing a new client| + +__minio.Options__ + +|Field | Type | Description | +|:--- |:--- | :--- | +| `opts.Creds` | _*credentials.Credentials_ | Access Credentials| +| `opts.Secure` | _bool_ | If 'true' API requests will be secure (HTTPS), and insecure (HTTP) otherwise | +| `opts.Region` | _string_ | region | +| `opts.BucketLookup` | _BucketLookupType_ | Bucket lookup type can be one of the following values | +| | | _minio.BucketLookupDNS_ | +| | | _minio.BucketLookupPath_ | +| | | _minio.BucketLookupAuto_ | +## 2. Bucket operations + + +### MakeBucket(bucketName, location string) error +Creates a new bucket. + +__Parameters__ + +| Param | Type | Description | +|---|---|---| +|`bucketName` | _string_ | Name of the bucket | +| `location` | _string_ | Region where the bucket is to be created. Default value is us-east-1. Other valid values are listed below. Note: When used with minio server, use the region specified in its config file (defaults to us-east-1).| +| | |us-east-1 | +| | |us-west-1 | +| | |us-west-2 | +| | |eu-west-1 | +| | | eu-central-1| +| | | ap-southeast-1| +| | | ap-northeast-1| +| | | ap-southeast-2| +| | | sa-east-1| + + +__Example__ + + +```go +err = minioClient.MakeBucket("mybucket", "us-east-1") +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully created mybucket.") +``` + + +### ListBuckets() ([]BucketInfo, error) +Lists all buckets. + +| Param | Type | Description | +|---|---|---| +|`bucketList` | _[]minio.BucketInfo_ | Lists of all buckets | + + +__minio.BucketInfo__ + +| Field | Type | Description | +|---|---|---| +|`bucket.Name` | _string_ | Name of the bucket | +|`bucket.CreationDate` | _time.Time_ | Date of bucket creation | + + +__Example__ + + +```go +buckets, err := minioClient.ListBuckets() +if err != nil { + fmt.Println(err) + return +} +for _, bucket := range buckets { + fmt.Println(bucket) +} +``` + + +### BucketExists(bucketName string) (found bool, err error) +Checks if a bucket exists. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`found` | _bool_ | Indicates whether bucket exists or not | +|`err` | _error_ | Standard Error | + + +__Example__ + + +```go +found, err := minioClient.BucketExists("mybucket") +if err != nil { + fmt.Println(err) + return +} +if found { + fmt.Println("Bucket found") +} +``` + + +### RemoveBucket(bucketName string) error +Removes a bucket, bucket should be empty to be successfully removed. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Example__ + + +```go +err = minioClient.RemoveBucket("mybucket") +if err != nil { + fmt.Println(err) + return +} +``` + + +### ListObjects(bucketName, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo +Lists objects in a bucket. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectPrefix` |_string_ | Prefix of objects to be listed | +|`recursive` | _bool_ |`true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. | +|`doneCh` | _chan struct{}_ | A message on this channel ends the ListObjects iterator. | + + +__Return Value__ + +|Param |Type |Description | +|:---|:---| :---| +|`objectInfo` | _chan minio.ObjectInfo_ |Read channel for all objects in the bucket, the object is of the format listed below: | + +__minio.ObjectInfo__ + +|Field |Type |Description | +|:---|:---| :---| +|`objectInfo.Key` | _string_ |Name of the object | +|`objectInfo.Size` | _int64_ |Size of the object | +|`objectInfo.ETag` | _string_ |MD5 checksum of the object | +|`objectInfo.LastModified` | _time.Time_ |Time when object was last modified | + + +```go +// Create a done channel to control 'ListObjects' go routine. +doneCh := make(chan struct{}) + +// Indicate to our routine to exit cleanly upon return. +defer close(doneCh) + +isRecursive := true +objectCh := minioClient.ListObjects("mybucket", "myprefix", isRecursive, doneCh) +for object := range objectCh { + if object.Err != nil { + fmt.Println(object.Err) + return + } + fmt.Println(object) +} +``` + + + +### ListObjectsV2(bucketName, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo +Lists objects in a bucket using the recommended listing API v2 + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +| `objectPrefix` |_string_ | Prefix of objects to be listed | +| `recursive` | _bool_ |`true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. | +|`doneCh` | _chan struct{}_ | A message on this channel ends the ListObjectsV2 iterator. | + + +__Return Value__ + +|Param |Type |Description | +|:---|:---| :---| +|`objectInfo` | _chan minio.ObjectInfo_ |Read channel for all the objects in the bucket, the object is of the format listed below: | + + +```go +// Create a done channel to control 'ListObjectsV2' go routine. +doneCh := make(chan struct{}) + +// Indicate to our routine to exit cleanly upon return. +defer close(doneCh) + +isRecursive := true +objectCh := minioClient.ListObjectsV2("mybucket", "myprefix", isRecursive, doneCh) +for object := range objectCh { + if object.Err != nil { + fmt.Println(object.Err) + return + } + fmt.Println(object) +} +``` + + +### ListIncompleteUploads(bucketName, prefix string, recursive bool, doneCh chan struct{}) <- chan ObjectMultipartInfo +Lists partially uploaded objects in a bucket. + + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +| `prefix` |_string_ | Prefix of objects that are partially uploaded | +| `recursive` | _bool_ |`true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. | +|`doneCh` | _chan struct{}_ | A message on this channel ends the ListenIncompleteUploads iterator. | + + +__Return Value__ + +|Param |Type |Description | +|:---|:---| :---| +|`multiPartInfo` | _chan minio.ObjectMultipartInfo_ |Emits multipart objects of the format listed below: | + +__minio.ObjectMultipartInfo__ + +|Field |Type |Description | +|:---|:---| :---| +|`multiPartObjInfo.Key` | _string_ |Name of incompletely uploaded object | +|`multiPartObjInfo.UploadID` | _string_ |Upload ID of incompletely uploaded object | +|`multiPartObjInfo.Size` | _int64_ |Size of incompletely uploaded object | + +__Example__ + + +```go +// Create a done channel to control 'ListObjects' go routine. +doneCh := make(chan struct{}) + +// Indicate to our routine to exit cleanly upon return. +defer close(doneCh) + +isRecursive := true // Recursively list everything at 'myprefix' +multiPartObjectCh := minioClient.ListIncompleteUploads("mybucket", "myprefix", isRecursive, doneCh) +for multiPartObject := range multiPartObjectCh { + if multiPartObject.Err != nil { + fmt.Println(multiPartObject.Err) + return + } + fmt.Println(multiPartObject) +} +``` + +## 3. Object operations + + +### GetObject(bucketName, objectName string, opts GetObjectOptions) (*Object, error) +Returns a stream of the object data. Most of the common errors occur when reading the stream. + + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`opts` | _minio.GetObjectOptions_ | Options for GET requests specifying additional options like encryption, If-Match | + + +__minio.GetObjectOptions__ + +|Field | Type | Description | +|:---|:---|:---| +| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | + +__Return Value__ + + +|Param |Type |Description | +|:---|:---| :---| +|`object` | _*minio.Object_ |_minio.Object_ represents object reader. It implements io.Reader, io.Seeker, io.ReaderAt and io.Closer interfaces. | + + +__Example__ + + +```go +object, err := minioClient.GetObject("mybucket", "myobject", minio.GetObjectOptions{}) +if err != nil { + fmt.Println(err) + return +} +localFile, err := os.Create("/tmp/local-file.jpg") +if err != nil { + fmt.Println(err) + return +} +if _, err = io.Copy(localFile, object); err != nil { + fmt.Println(err) + return +} +``` + + +### FGetObject(bucketName, objectName, filePath string, opts GetObjectOptions) error +Downloads and saves the object as a file in the local filesystem. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`filePath` | _string_ |Path to download object to | +|`opts` | _minio.GetObjectOptions_ | Options for GET requests specifying additional options like encryption, If-Match | + + +__Example__ + + +```go +err = minioClient.FGetObject("mybucket", "myobject", "/tmp/myobject", minio.GetObjectOptions{}) +if err != nil { + fmt.Println(err) + return +} +``` + +### GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) +Identical to GetObject operation, but accepts a context for request cancellation. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`ctx` | _context.Context_ |Request context | +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`opts` | _minio.GetObjectOptions_ | Options for GET requests specifying additional options like encryption, If-Match | + + +__Return Value__ + + +|Param |Type |Description | +|:---|:---| :---| +|`object` | _*minio.Object_ |_minio.Object_ represents object reader. It implements io.Reader, io.Seeker, io.ReaderAt and io.Closer interfaces. | + + +__Example__ + + +```go +ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) +defer cancel() + +object, err := minioClient.GetObjectWithContext(ctx, "mybucket", "myobject", minio.GetObjectOptions{}) +if err != nil { + fmt.Println(err) + return +} + +localFile, err := os.Create("/tmp/local-file.jpg") +if err != nil { + fmt.Println(err) + return +} + +if _, err = io.Copy(localFile, object); err != nil { + fmt.Println(err) + return +} +``` + + +### FGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error +Identical to FGetObject operation, but allows request cancellation. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`ctx` | _context.Context_ |Request context | +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`filePath` | _string_ |Path to download object to | +|`opts` | _minio.GetObjectOptions_ | Options for GET requests specifying additional options like encryption, If-Match | + + +__Example__ + + +```go +ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) +defer cancel() + +err = minioClient.FGetObjectWithContext(ctx, "mybucket", "myobject", "/tmp/myobject", minio.GetObjectOptions{}) +if err != nil { + fmt.Println(err) + return +} +``` + + +### PutObject(bucketName, objectName string, reader io.Reader, objectSize int64,opts PutObjectOptions) (n int, err error) +Uploads objects that are less than 64MiB in a single PUT operation. For objects that are greater than 64MiB in size, PutObject seamlessly uploads the object as parts of 64MiB or more depending on the actual file size. The max upload size for an object is 5TB. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`reader` | _io.Reader_ |Any Go type that implements io.Reader | +|`objectSize`| _int64_ |Size of the object being uploaded. Pass -1 if stream size is unknown | +|`opts` | _minio.PutObjectOptions_ | Allows user to set optional custom metadata, content headers, encryption keys and number of threads for multipart upload operation. | + +__minio.PutObjectOptions__ + +|Field | Type | Description | +|:--- |:--- | :--- | +| `opts.UserMetadata` | _map[string]string_ | Map of user metadata| +| `opts.Progress` | _io.Reader_ | Reader to fetch progress of an upload | +| `opts.ContentType` | _string_ | Content type of object, e.g "application/text" | +| `opts.ContentEncoding` | _string_ | Content encoding of object, e.g "gzip" | +| `opts.ContentDisposition` | _string_ | Content disposition of object, "inline" | +| `opts.ContentLanguage` | _string_ | Content language of object, e.g "French" | +| `opts.CacheControl` | _string_ | Used to specify directives for caching mechanisms in both requests and responses e.g "max-age=600"| +| `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | +| `opts.StorageClass` | _string_ | Specify storage class for the object. Supported values for Minio server are `REDUCED_REDUNDANCY` and `STANDARD` | +| `opts.WebsiteRedirectLocation` | _string_ | Specify a redirect for the object, to another object in the same bucket or to a external URL. | + +__Example__ + + +```go +file, err := os.Open("my-testfile") +if err != nil { + fmt.Println(err) + return +} +defer file.Close() + +fileStat, err := file.Stat() +if err != nil { + fmt.Println(err) + return +} + +n, err := minioClient.PutObject("mybucket", "myobject", file, fileStat.Size(), minio.PutObjectOptions{ContentType:"application/octet-stream"}) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully uploaded bytes: ", n) +``` + +API methods PutObjectWithSize, PutObjectWithMetadata, PutObjectStreaming, and PutObjectWithProgress available in minio-go SDK release v3.0.3 are replaced by the new PutObject call variant that accepts a pointer to PutObjectOptions struct. + + +### PutObjectWithContext(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, opts PutObjectOptions) (n int, err error) +Identical to PutObject operation, but allows request cancellation. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`ctx` | _context.Context_ |Request context | +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`reader` | _io.Reader_ |Any Go type that implements io.Reader | +|`objectSize`| _int64_ | size of the object being uploaded. Pass -1 if stream size is unknown | +|`opts` | _minio.PutObjectOptions_ |Pointer to struct that allows user to set optional custom metadata, content-type, content-encoding, content-disposition, content-language and cache-control headers, pass encryption module for encrypting objects, and optionally configure number of threads for multipart put operation. | + + +__Example__ + + +```go +ctx, cancel := context.WithTimeout(context.Background(), 10 * time.Second) +defer cancel() + +file, err := os.Open("my-testfile") +if err != nil { + fmt.Println(err) + return +} +defer file.Close() + +fileStat, err := file.Stat() +if err != nil { + fmt.Println(err) + return +} + +n, err := minioClient.PutObjectWithContext(ctx, "my-bucketname", "my-objectname", file, fileStat.Size(), minio.PutObjectOptions{ + ContentType: "application/octet-stream", +}) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully uploaded bytes: ", n) +``` + + +### CopyObject(dst DestinationInfo, src SourceInfo) error +Create or replace an object through server-side copying of an existing object. It supports conditional copying, copying a part of an object and server-side encryption of destination and decryption of source. See the `SourceInfo` and `DestinationInfo` types for further details. + +To copy multiple source objects into a single destination object see the `ComposeObject` API. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`dst` | _minio.DestinationInfo_ |Argument describing the destination object | +|`src` | _minio.SourceInfo_ |Argument describing the source object | + + +__Example__ + + +```go +// Use-case 1: Simple copy object with no conditions. +// Source object +src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) + +// Destination object +dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + +```go +// Use-case 2: +// Copy object with copy-conditions, and copying only part of the source object. +// 1. that matches a given ETag +// 2. and modified after 1st April 2014 +// 3. but unmodified since 23rd April 2014 +// 4. copy only first 1MiB of object. + +// Source object +src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) + +// Set matching ETag condition, copy object which matches the following ETag. +src.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") + +// Set modified condition, copy object modified since 2014 April 1. +src.SetModifiedSinceCond(time.Date(2014, time.April, 1, 0, 0, 0, 0, time.UTC)) + +// Set unmodified condition, copy object unmodified since 2014 April 23. +src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 23, 0, 0, 0, 0, time.UTC)) + +// Set copy-range of only first 1MiB of file. +src.SetRange(0, 1024*1024-1) + +// Destination object +dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + + +### ComposeObject(dst minio.DestinationInfo, srcs []minio.SourceInfo) error +Create an object by concatenating a list of source objects using server-side copying. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---|:---| +|`dst` | _minio.DestinationInfo_ |Struct with info about the object to be created. | +|`srcs` | _[]minio.SourceInfo_ |Slice of struct with info about source objects to be concatenated in order. | + + +__Example__ + + +```go +// Prepare source decryption key (here we assume same key to +// decrypt all source objects.) +sseSrc := encrypt.DefaultPBKDF([]byte("password"), []byte("salt")) + +// Source objects to concatenate. We also specify decryption +// key for each +src1 := minio.NewSourceInfo("bucket1", "object1", sseSrc) +src1.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") + +src2 := minio.NewSourceInfo("bucket2", "object2", sseSrc) +src2.SetMatchETagCond("f8ef9c385918b653a31624deb84149d2") + +src3 := minio.NewSourceInfo("bucket3", "object3", sseSrc) +src3.SetMatchETagCond("5918b653a31624deb84149d2f8ef9c38") + +// Create slice of sources. +srcs := []minio.SourceInfo{src1, src2, src3} + +// Prepare destination encryption key +sseDst := encrypt.DefaultPBKDF([]byte("new-password"), []byte("new-salt")) + +// Create destination info +dst, err := minio.NewDestinationInfo("bucket", "object", sseDst, nil) +if err != nil { + fmt.Println(err) + return +} + +// Compose object call by concatenating multiple source files. +err = minioClient.ComposeObject(dst, srcs) +if err != nil { + fmt.Println(err) + return +} + +fmt.Println("Composed object successfully.") +``` + + +### NewSourceInfo(bucket, object string, decryptSSEC *SSEInfo) SourceInfo +Construct a `SourceInfo` object that can be used as the source for server-side copying operations like `CopyObject` and `ComposeObject`. This object can be used to set copy-conditions on the source. + +__Parameters__ + +| Param | Type | Description | +| :--- | :--- | :--- | +| `bucket` | _string_ | Name of the source bucket | +| `object` | _string_ | Name of the source object | +| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | + +__Example__ + +```go +// No decryption parameter. +src := minio.NewSourceInfo("bucket", "object", nil) + +// Destination object +dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + +```go +// With decryption parameter. +sseSrc := encrypt.DefaultPBKDF([]byte("password"), []byte("salt")) +src := minio.NewSourceInfo("bucket", "object", sseSrc) + +// Destination object +dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + + +### NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, userMeta map[string]string) (DestinationInfo, error) +Construct a `DestinationInfo` object that can be used as the destination object for server-side copying operations like `CopyObject` and `ComposeObject`. + +__Parameters__ + +| Param | Type | Description | +| :--- | :--- | :--- | +| `bucket` | _string_ | Name of the destination bucket | +| `object` | _string_ | Name of the destination object | +| `sse` | _*encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go) | | +| `userMeta` | _map[string]string_ | User metadata to be set on the destination. If nil, with only one source, user-metadata is copied from source. | + +__Example__ + +```go +// No encryption parameter. +src := minio.NewSourceInfo("bucket", "object", nil) +dst, err := minio.NewDestinationInfo("bucket", "object", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + +```go +src := minio.NewSourceInfo("bucket", "object", nil) + +// With encryption parameter. +sseDst := encrypt.DefaultPBKDF([]byte("password"), []byte("salt")) +dst, err := minio.NewDestinationInfo("bucket", "object", sseDst, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + + +### FPutObject(bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) +Uploads contents from a file to objectName. + +FPutObject uploads objects that are less than 64MiB in a single PUT operation. For objects that are greater than the 64MiB in size, FPutObject seamlessly uploads the object in chunks of 64MiB or more depending on the actual file size. The max upload size for an object is 5TB. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`filePath` | _string_ |Path to file to be uploaded | +|`opts` | _minio.PutObjectOptions_ |Pointer to struct that allows user to set optional custom metadata, content-type, content-encoding, content-disposition, content-language and cache-control headers, pass encryption module for encrypting objects, and optionally configure number of threads for multipart put operation. | + + +__Example__ + + +```go +n, err := minioClient.FPutObject("my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ + ContentType: "application/csv", +}); +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully uploaded bytes: ", n) +``` + + +### FPutObjectWithContext(ctx context.Context, bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) +Identical to FPutObject operation, but allows request cancellation. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`ctx` | _context.Context_ |Request context | +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`filePath` | _string_ |Path to file to be uploaded | +|`opts` | _minio.PutObjectOptions_ |Pointer to struct that allows user to set optional custom metadata, content-type, content-encoding,content-disposition and cache-control headers, pass encryption module for encrypting objects, and optionally configure number of threads for multipart put operation. | + +__Example__ + + +```go +ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) +defer cancel() + +n, err := minioClient.FPutObjectWithContext(ctx, "mybucket", "myobject.csv", "/tmp/otherobject.csv", minio.PutObjectOptions{ContentType:"application/csv"}) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully uploaded bytes: ", n) +``` + + +### StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) +Fetch metadata of an object. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`opts` | _minio.StatObjectOptions_ | Options for GET info/stat requests specifying additional options like encryption, If-Match | + + +__Return Value__ + +|Param |Type |Description | +|:---|:---| :---| +|`objInfo` | _minio.ObjectInfo_ |Object stat information | + + +__minio.ObjectInfo__ + +|Field |Type |Description | +|:---|:---| :---| +|`objInfo.LastModified` | _time.Time_ |Time when object was last modified | +|`objInfo.ETag` | _string_ |MD5 checksum of the object| +|`objInfo.ContentType` | _string_ |Content type of the object| +|`objInfo.Size` | _int64_ |Size of the object| + + +__Example__ + + +```go +objInfo, err := minioClient.StatObject("mybucket", "myobject", minio.StatObjectOptions{}) +if err != nil { + fmt.Println(err) + return +} +fmt.Println(objInfo) +``` + + +### RemoveObject(bucketName, objectName string) error +Removes an object. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | + + +```go +err = minioClient.RemoveObject("mybucket", "myobject") +if err != nil { + fmt.Println(err) + return +} +``` + + +### RemoveObjects(bucketName string, objectsCh chan string) (errorCh <-chan RemoveObjectError) +Removes a list of objects obtained from an input channel. The call sends a delete request to the server up to 1000 objects at a time. The errors observed are sent over the error channel. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectsCh` | _chan string_ | Channel of objects to be removed | + + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`errorCh` | _<-chan minio.RemoveObjectError_ | Receive-only channel of errors observed during deletion. | + + +```go +objectsCh := make(chan string) + +// Send object names that are needed to be removed to objectsCh +go func() { + defer close(objectsCh) + // List all objects from a bucket-name with a matching prefix. + for object := range minioClient.ListObjects("my-bucketname", "my-prefixname", true, nil) { + if object.Err != nil { + log.Fatalln(object.Err) + } + objectsCh <- object.Key + } +}() + +for rErr := range minioClient.RemoveObjects("mybucket", objectsCh) { + fmt.Println("Error detected during deletion: ", rErr) +} +``` + + +### RemoveObjectsWithContext(ctx context.Context, bucketName string, objectsCh chan string) (errorCh <-chan RemoveObjectError) +*Identical to RemoveObjects operation, but accepts a context for request cancellation.* + +Parameters + +|Param |Type |Description | +|:---|:---| :---| +|`ctx` | _context.Context_ |Request context | +|`bucketName` | _string_ |Name of the bucket | +|`objectsCh` | _chan string_ | Channel of objects to be removed | + + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`errorCh` | _<-chan minio.RemoveObjectError_ | Receive-only channel of errors observed during deletion. | + +```go +objectsCh := make(chan string) +ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) +defer cancel() + +// Send object names that are needed to be removed to objectsCh +go func() { + defer close(objectsCh) + // List all objects from a bucket-name with a matching prefix. + for object := range minioClient.ListObjects("my-bucketname", "my-prefixname", true, nil) { + if object.Err != nil { + log.Fatalln(object.Err) + } + objectsCh <- object.Key + } +}() + +for rErr := range minioClient.RemoveObjects(ctx, "my-bucketname", objectsCh) { + fmt.Println("Error detected during deletion: ", rErr) +} +``` + +### SelectObjectContent(ctx context.Context, bucketName string, objectsName string, expression string, options SelectObjectOptions) *SelectResults +Parameters + +|Param |Type |Description | +|:---|:---| :---| +|`ctx` | _context.Context_ |Request context | +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`options` | _SelectObjectOptions_ | Query Options | + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`SelectResults` | _SelectResults_ | Is an io.ReadCloser object which can be directly passed to csv.NewReader for processing output. | + +```go + // Initialize minio client object. + minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, useSSL) + if err != nil { + log.Fatalln(err) + } + + opts := minio.SelectObjectOptions{ + Expression: "select count(*) from s3object", + ExpressionType: minio.QueryExpressionTypeSQL, + InputSerialization: minio.SelectObjectInputSerialization{ + CompressionType: minio.SelectCompressionNONE, + CSV: &minio.CSVInputOptions{ + FileHeaderInfo: minio.CSVFileHeaderInfoNone, + RecordDelimiter: "\n", + FieldDelimiter: ",", + }, + }, + OutputSerialization: minio.SelectObjectOutputSerialization{ + CSV: &minio.CSVOutputOptions{ + RecordDelimiter: "\n", + FieldDelimiter: ",", + }, + }, + } + + reader, err := s3Client.SelectObjectContent(context.Background(), "mycsvbucket", "mycsv.csv", opts) + if err != nil { + log.Fatalln(err) + } + defer reader.Close() + + if _, err := io.Copy(os.Stdout, reader); err != nil { + log.Fatalln(err) + } +``` + + +### RemoveIncompleteUpload(bucketName, objectName string) error +Removes a partially uploaded object. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | + +__Example__ + + +```go +err = minioClient.RemoveIncompleteUpload("mybucket", "myobject") +if err != nil { + fmt.Println(err) + return +} +``` + +## 5. Presigned operations + + +### PresignedGetObject(bucketName, objectName string, expiry time.Duration, reqParams url.Values) (*url.URL, error) +Generates a presigned URL for HTTP GET operations. Browsers/Mobile clients may point to this URL to directly download objects even if the bucket is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The default expiry is set to 7 days. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`expiry` | _time.Duration_ |Expiry of presigned URL in seconds | +|`reqParams` | _url.Values_ |Additional response header overrides supports _response-expires_, _response-content-type_, _response-cache-control_, _response-content-disposition_. | + + +__Example__ + + +```go +// Set request parameters for content-disposition. +reqParams := make(url.Values) +reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") + +// Generates a presigned url which expires in a day. +presignedURL, err := minioClient.PresignedGetObject("mybucket", "myobject", time.Second * 24 * 60 * 60, reqParams) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully generated presigned URL", presignedURL) +``` + + +### PresignedPutObject(bucketName, objectName string, expiry time.Duration) (*url.URL, error) +Generates a presigned URL for HTTP PUT operations. Browsers/Mobile clients may point to this URL to upload objects directly to a bucket even if it is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The default expiry is set to 7 days. + +NOTE: you can upload to S3 only with specified object name. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`expiry` | _time.Duration_ |Expiry of presigned URL in seconds | + + +__Example__ + + +```go +// Generates a url which expires in a day. +expiry := time.Second * 24 * 60 * 60 // 1 day. +presignedURL, err := minioClient.PresignedPutObject("mybucket", "myobject", expiry) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully generated presigned URL", presignedURL) +``` + + +### PresignedHeadObject(bucketName, objectName string, expiry time.Duration, reqParams url.Values) (*url.URL, error) +Generates a presigned URL for HTTP HEAD operations. Browsers/Mobile clients may point to this URL to directly get metadata from objects even if the bucket is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The default expiry is set to 7 days. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`expiry` | _time.Duration_ |Expiry of presigned URL in seconds | +|`reqParams` | _url.Values_ |Additional response header overrides supports _response-expires_, _response-content-type_, _response-cache-control_, _response-content-disposition_. | + + +__Example__ + + +```go +// Set request parameters for content-disposition. +reqParams := make(url.Values) +reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") + +// Generates a presigned url which expires in a day. +presignedURL, err := minioClient.PresignedHeadObject("mybucket", "myobject", time.Second * 24 * 60 * 60, reqParams) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully generated presigned URL", presignedURL) +``` + + +### PresignedPostPolicy(PostPolicy) (*url.URL, map[string]string, error) +Allows setting policy conditions to a presigned URL for POST operations. Policies such as bucket name to receive object uploads, key name prefixes, expiry policy may be set. + +```go +// Initialize policy condition config. +policy := minio.NewPostPolicy() + +// Apply upload policy restrictions: +policy.SetBucket("mybucket") +policy.SetKey("myobject") +policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days + +// Only allow 'png' images. +policy.SetContentType("image/png") + +// Only allow content size in range 1KB to 1MB. +policy.SetContentLengthRange(1024, 1024*1024) + +// Add a user metadata using the key "custom" and value "user" +policy.SetUserMetadata("custom", "user") + +// Get the POST form key/value object: +url, formData, err := minioClient.PresignedPostPolicy(policy) +if err != nil { + fmt.Println(err) + return +} + +// POST your content from the command line using `curl` +fmt.Printf("curl ") +for k, v := range formData { + fmt.Printf("-F %s=%s ", k, v) +} +fmt.Printf("-F file=@/etc/bash.bashrc ") +fmt.Printf("%s\n", url) +``` + +## 6. Bucket policy/notification operations + + +### SetBucketPolicy(bucketname, policy string) error +Set access permissions on bucket or an object prefix. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket| +|`policy` | _string_ |Policy to be set | + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + +```go +policy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::my-bucketname/*"],"Sid": ""}]}` + +err = minioClient.SetBucketPolicy("my-bucketname", policy) +if err != nil { + fmt.Println(err) + return +} +``` + + +### GetBucketPolicy(bucketName) (policy string, error) +Get access permissions on a bucket or a prefix. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`policy` | _string_ |Policy returned from the server | +|`err` | _error_ |Standard Error | + +__Example__ + +```go +policy, err := minioClient.GetBucketPolicy("my-bucketname") +if err != nil { + log.Fatalln(err) +} +``` + + +### GetBucketNotification(bucketName string) (BucketNotification, error) +Get notification configuration on a bucket. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketNotification` | _minio.BucketNotification_ |structure which holds all notification configurations| +|`err` | _error_ |Standard Error | + +__Example__ + + +```go +bucketNotification, err := minioClient.GetBucketNotification("mybucket") +if err != nil { + fmt.Println("Failed to get bucket notification configurations for mybucket", err) + return +} + +for _, queueConfig := range bucketNotification.QueueConfigs { + for _, e := range queueConfig.Events { + fmt.Println(e + " event is enabled") + } +} +``` + + +### SetBucketNotification(bucketName string, bucketNotification BucketNotification) error +Set a new bucket notification on a bucket. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`bucketNotification` | _minio.BucketNotification_ |Represents the XML to be sent to the configured web service | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + + +```go +queueArn := minio.NewArn("aws", "sqs", "us-east-1", "804605494417", "PhotoUpdate") + +queueConfig := minio.NewNotificationConfig(queueArn) +queueConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) +queueConfig.AddFilterPrefix("photos/") +queueConfig.AddFilterSuffix(".jpg") + +bucketNotification := minio.BucketNotification{} +bucketNotification.AddQueue(queueConfig) + +err = minioClient.SetBucketNotification("mybucket", bucketNotification) +if err != nil { + fmt.Println("Unable to set the bucket notification: ", err) + return +} +``` + + +### RemoveAllBucketNotification(bucketName string) error +Remove all configured bucket notifications on a bucket. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + + +```go +err = minioClient.RemoveAllBucketNotification("mybucket") +if err != nil { + fmt.Println("Unable to remove bucket notifications.", err) + return +} +``` + + +### ListenBucketNotification(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo +ListenBucketNotification API receives bucket notification events through the notification channel. The returned notification channel has two fields 'Records' and 'Err'. + +- 'Records' holds the notifications received from the server. +- 'Err' indicates any error while processing the received notifications. + +NOTE: Notification channel is closed at the first occurrence of an error. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ | Bucket to listen notifications on | +|`prefix` | _string_ | Object key prefix to filter notifications for | +|`suffix` | _string_ | Object key suffix to filter notifications for | +|`events` | _[]string_ | Enables notifications for specific event types | +|`doneCh` | _chan struct{}_ | A message on this channel ends the ListenBucketNotification iterator | + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`notificationInfo` | _chan minio.NotificationInfo_ | Channel of bucket notifications | + +__minio.NotificationInfo__ + +|Field |Type |Description | +|`notificationInfo.Records` | _[]minio.NotificationEvent_ | Collection of notification events | +|`notificationInfo.Err` | _error_ | Carries any error occurred during the operation (Standard Error) | + + +__Example__ + + +```go +// Create a done channel to control 'ListenBucketNotification' go routine. +doneCh := make(chan struct{}) + +// Indicate a background go-routine to exit cleanly upon return. +defer close(doneCh) + +// Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. +for notificationInfo := range minioClient.ListenBucketNotification("mybucket", "myprefix/", ".mysuffix", []string{ + "s3:ObjectCreated:*", + "s3:ObjectAccessed:*", + "s3:ObjectRemoved:*", + }, doneCh) { + if notificationInfo.Err != nil { + fmt.Println(notificationInfo.Err) + } + fmt.Println(notificationInfo) +} +``` + + +### SetBucketLifecycle(bucketname, lifecycle string) error +Set lifecycle on bucket or an object prefix. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket| +|`lifecycle` | _string_ |Lifecycle to be set | + +__Return Values__ + +|Param |Type |Description | +|:---|:---| :---| +|`err` | _error_ |Standard Error | + +__Example__ + +```go +lifecycle := ` + + expire-bucket + + Enabled + + 365 + + +` + +err = minioClient.SetBucketLifecycle("my-bucketname", lifecycle) +if err != nil { + fmt.Println(err) + return +} +``` + + +### GetBucketLifecycle(bucketName) (lifecycle string, error) +Get lifecycle on a bucket or a prefix. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | + +__Return Values__ + + +|Param |Type |Description | +|:---|:---| :---| +|`lifecycle` | _string_ |Lifecycle returned from the server | +|`err` | _error_ |Standard Error | + +__Example__ + +```go +lifecycle, err := minioClient.GetBucketLifecycle("my-bucketname") +if err != nil { + log.Fatalln(err) +} +``` + +## 7. Client custom settings + + +### SetAppInfo(appName, appVersion string) +Add custom application details to User-Agent. + +__Parameters__ + +| Param | Type | Description | +|---|---|---| +|`appName` | _string_ | Name of the application performing the API requests. | +| `appVersion`| _string_ | Version of the application performing the API requests. | + + +__Example__ + + +```go +// Set Application name and version to be used in subsequent API requests. +minioClient.SetAppInfo("myCloudApp", "1.0.0") +``` + + +### SetCustomTransport(customHTTPTransport http.RoundTripper) +Overrides default HTTP transport. This is usually needed for debugging or for adding custom TLS certificates. + +__Parameters__ + +| Param | Type | Description | +|---|---|---| +|`customHTTPTransport` | _http.RoundTripper_ | Custom transport e.g, to trace API requests and responses for debugging purposes.| + + + +### TraceOn(outputStream io.Writer) +Enables HTTP tracing. The trace is written to the io.Writer provided. If outputStream is nil, trace is written to os.Stdout. + +__Parameters__ + +| Param | Type | Description | +|---|---|---| +|`outputStream` | _io.Writer_ | HTTP trace is written into outputStream.| + + + +### TraceOff() +Disables HTTP tracing. + + +### SetS3TransferAccelerate(acceleratedEndpoint string) +Set AWS S3 transfer acceleration endpoint for all API requests hereafter. +NOTE: This API applies only to AWS S3 and is a no operation for S3 compatible object storage services. + +__Parameters__ + +| Param | Type | Description | +|---|---|---| +|`acceleratedEndpoint` | _string_ | Set to new S3 transfer acceleration endpoint.| + + +## 8. Explore Further + +- [Build your own Go Music Player App example](https://docs.minio.io/docs/go-music-player-app) diff --git a/docs/checker.go.template b/docs/checker.go.template new file mode 100644 index 0000000..2e0f13a --- /dev/null +++ b/docs/checker.go.template @@ -0,0 +1,21 @@ +package main + +import ( + "fmt" + + "github.com/minio/minio-go" +) + +func main() { + // Use a secure connection. + ssl := true + + // Initialize minio client object. + minioClient, err := minio.New("play.minio.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) + if err != nil { + fmt.Println(err) + return + } + + {{.Text}} +} diff --git a/docs/validator.go b/docs/validator.go new file mode 100644 index 0000000..7d5cbaa --- /dev/null +++ b/docs/validator.go @@ -0,0 +1,227 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" + + "github.com/a8m/mark" + "github.com/gernest/wow" + "github.com/gernest/wow/spin" + "github.com/minio/cli" +) + +func init() { + // Validate go binary. + if _, err := exec.LookPath("go"); err != nil { + panic(err) + } +} + +var globalFlags = []cli.Flag{ + cli.StringFlag{ + Name: "m", + Value: "API.md", + Usage: "Path to markdown api documentation.", + }, + cli.StringFlag{ + Name: "t", + Value: "checker.go.template", + Usage: "Template used for generating the programs.", + }, + cli.IntFlag{ + Name: "skip", + Value: 2, + Usage: "Skip entries before validating the code.", + }, +} + +func runGofmt(path string) (msg string, err error) { + cmdArgs := []string{"-s", "-w", "-l", path} + cmd := exec.Command("gofmt", cmdArgs...) + stdoutStderr, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + return string(stdoutStderr), nil +} + +func runGoImports(path string) (msg string, err error) { + cmdArgs := []string{"-w", path} + cmd := exec.Command("goimports", cmdArgs...) + stdoutStderr, err := cmd.CombinedOutput() + if err != nil { + return string(stdoutStderr), err + } + return string(stdoutStderr), nil +} + +func runGoBuild(path string) (msg string, err error) { + // Go build the path. + cmdArgs := []string{"build", "-o", "/dev/null", path} + cmd := exec.Command("go", cmdArgs...) + stdoutStderr, err := cmd.CombinedOutput() + if err != nil { + return string(stdoutStderr), err + } + return string(stdoutStderr), nil +} + +func validatorAction(ctx *cli.Context) error { + if !ctx.IsSet("m") || !ctx.IsSet("t") { + return nil + } + docPath := ctx.String("m") + var err error + docPath, err = filepath.Abs(docPath) + if err != nil { + return err + } + data, err := ioutil.ReadFile(docPath) + if err != nil { + return err + } + + templatePath := ctx.String("t") + templatePath, err = filepath.Abs(templatePath) + if err != nil { + return err + } + + skipEntries := ctx.Int("skip") + m := mark.New(string(data), &mark.Options{ + Gfm: true, // Github markdown support is enabled by default. + }) + + t, err := template.ParseFiles(templatePath) + if err != nil { + return err + } + + tmpDir, err := ioutil.TempDir("", "md-verifier") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + entryN := 1 + for i := mark.NodeText; i < mark.NodeCheckbox; i++ { + if mark.NodeCode != mark.NodeType(i) { + m.AddRenderFn(mark.NodeType(i), func(node mark.Node) (s string) { + return "" + }) + continue + } + m.AddRenderFn(mark.NodeCode, func(node mark.Node) (s string) { + p, ok := node.(*mark.CodeNode) + if !ok { + return + } + p.Text = strings.NewReplacer("<", "<", ">", ">", """, `"`, "&", "&").Replace(p.Text) + if skipEntries > 0 { + skipEntries-- + return + } + + testFilePath := filepath.Join(tmpDir, "example.go") + w, werr := os.Create(testFilePath) + if werr != nil { + panic(werr) + } + t.Execute(w, p) + w.Sync() + w.Close() + entryN++ + + msg, err := runGofmt(testFilePath) + if err != nil { + fmt.Printf("Failed running gofmt on %s, with (%s):(%s)\n", testFilePath, msg, err) + os.Exit(-1) + } + + msg, err = runGoImports(testFilePath) + if err != nil { + fmt.Printf("Failed running gofmt on %s, with (%s):(%s)\n", testFilePath, msg, err) + os.Exit(-1) + } + + msg, err = runGoBuild(testFilePath) + if err != nil { + fmt.Printf("Failed running gobuild on %s, with (%s):(%s)\n", testFilePath, msg, err) + fmt.Printf("Code with possible issue in %s:\n%s", docPath, p.Text) + fmt.Printf("To test `go build %s`\n", testFilePath) + os.Exit(-1) + } + + // Once successfully built remove the test file + os.Remove(testFilePath) + return + }) + } + + w := wow.New(os.Stdout, spin.Get(spin.Moon), fmt.Sprintf(" Running validation tests in %s", tmpDir)) + + w.Start() + // Render markdown executes our checker on each code blocks. + _ = m.Render() + w.PersistWith(spin.Get(spin.Runner), " Successfully finished tests") + w.Stop() + + return nil +} + +func main() { + app := cli.NewApp() + app.Action = validatorAction + app.HideVersion = true + app.HideHelpCommand = true + app.Usage = "Validates code block sections inside API.md" + app.Author = "Minio.io" + app.Flags = globalFlags + // Help template for validator + app.CustomAppHelpTemplate = `NAME: + {{.Name}} - {{.Usage}} + +USAGE: + {{.Name}} {{if .VisibleFlags}}[FLAGS] {{end}}COMMAND{{if .VisibleFlags}} [COMMAND FLAGS | -h]{{end}} [ARGUMENTS...] + +COMMANDS: + {{range .VisibleCommands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} + {{end}}{{if .VisibleFlags}} +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} +TEMPLATE: + Validator uses Go's 'text/template' formatting so you need to ensure + your template is formatted correctly, check 'docs/checker.go.template' + +USAGE: + go run docs/validator.go -m docs/API.md -t /tmp/mycode.go.template + +` + app.Run(os.Args) + +} diff --git a/docs/zh_CN/API.md b/docs/zh_CN/API.md new file mode 100644 index 0000000..ca80586 --- /dev/null +++ b/docs/zh_CN/API.md @@ -0,0 +1,1786 @@ +# Minio Go Client API文档 [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) + +## 初使化Minio Client对象。 + +## Minio + +```go +package main + +import ( + "fmt" + + "github.com/minio/minio-go" +) + +func main() { + // 使用ssl + ssl := true + + // 初使化minio client对象。 + minioClient, err := minio.New("play.minio.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) + if err != nil { + fmt.Println(err) + return + } +} +``` + +## AWS S3 + +```go +package main + +import ( + "fmt" + + "github.com/minio/minio-go" +) + +func main() { + // 使用ssl + ssl := true + + // 初使化minio client对象。 + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ssl) + if err != nil { + fmt.Println(err) + return + } +} +``` + +| 操作存储桶 | 操作对象 | 操作加密对象 | Presigned操作 | 存储桶策略/通知 | 客户端自定义设置 | +| :--- | :--- | :--- | :--- | :--- | :--- | +| [`MakeBucket`](#MakeBucket) | [`GetObject`](#GetObject) | [`NewSymmetricKey`](#NewSymmetricKey) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | +| [`ListBuckets`](#ListBuckets) | [`PutObject`](#PutObject) | [`NewAsymmetricKey`](#NewAsymmetricKey) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | +| [`BucketExists`](#BucketExists) | [`CopyObject`](#CopyObject) | [`GetEncryptedObject`](#GetEncryptedObject) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOn`](#TraceOn) | +| [`RemoveBucket`](#RemoveBucket) | [`StatObject`](#StatObject) | [`PutEncryptedObject`](#PutEncryptedObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`TraceOff`](#TraceOff) | +| [`ListObjects`](#ListObjects) | [`RemoveObject`](#RemoveObject) | [`NewSSEInfo`](#NewSSEInfo) | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | +| [`ListObjectsV2`](#ListObjectsV2) | [`RemoveObjects`](#RemoveObjects) | [`FPutEncryptedObject`](#FPutEncryptedObject) | | [`ListenBucketNotification`](#ListenBucketNotification) | | +| [`ListIncompleteUploads`](#ListIncompleteUploads) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | | | +| | [`FPutObject`](#FPutObject) | | | | | +| | [`FGetObject`](#FGetObject) | | | | | +| | [`ComposeObject`](#ComposeObject) | | | | | +| | [`NewSourceInfo`](#NewSourceInfo) | | | | | +| | [`NewDestinationInfo`](#NewDestinationInfo) | | | | | +| | [`PutObjectWithContext`](#PutObjectWithContext) | | | | +| | [`GetObjectWithContext`](#GetObjectWithContext) | | | | +| | [`FPutObjectWithContext`](#FPutObjectWithContext) | | | | +| | [`FGetObjectWithContext`](#FGetObjectWithContext) | | | | +## 1. 构造函数 + + +### New(endpoint, accessKeyID, secretAccessKey string, ssl bool) (*Client, error) +初使化一个新的client对象。 + +__参数__ + +|参数 | 类型 |描述 | +|:---|:---| :---| +|`endpoint` | _string_ |S3兼容对象存储服务endpoint | +|`accessKeyID` |_string_ |对象存储的Access key | +|`secretAccessKey` | _string_ |对象存储的Secret key | +|`ssl` | _bool_ |true代表使用HTTPS | + +### NewWithRegion(endpoint, accessKeyID, secretAccessKey string, ssl bool, region string) (*Client, error) +初使化minio client,带有region配置。和New()不同的是，NewWithRegion避免了bucket-location操作，所以会快那么一丢丢。如果你的应用只使用一个region的话可以用这个方法。 + +__参数__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`endpoint` | _string_ |S3兼容对象存储服务endpoint | +|`accessKeyID` |_string_ |对象存储的Access key | +|`secretAccessKey` | _string_ |对象存储的Secret key | +|`ssl` | _bool_ |true代表使用HTTPS | +|`region`| _string_ | 对象存储的region | + +## 2. 操作存储桶 + + +### MakeBucket(bucketName, location string) error +创建一个存储桶。 + +__参数__ + +| 参数 | 类型 | 描述 | +|---|---|---| +|`bucketName` | _string_ | 存储桶名称 | +| `location` | _string_ | 存储桶被创建的region(地区)，默认是us-east-1(美国东一区)，下面列举的是其它合法的值。注意：如果用的是minio服务的话，resion是在它的配置文件中，（默认是us-east-1）。| +| | |us-east-1 | +| | |us-west-1 | +| | |us-west-2 | +| | |eu-west-1 | +| | | eu-central-1| +| | | ap-southeast-1| +| | | ap-northeast-1| +| | | ap-southeast-2| +| | | sa-east-1| + + +__示例__ + + +```go +err = minioClient.MakeBucket("mybucket", "us-east-1") +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully created mybucket.") +``` + + +### ListBuckets() ([]BucketInfo, error) +列出所有的存储桶。 + +| 参数 | 类型 | 描述 | +|---|---|---| +|`bucketList` | _[]minio.BucketInfo_ | 所有存储桶的list。 | + + +__minio.BucketInfo__ + +| 参数 | 类型 | 描述 | +|---|---|---| +|`bucket.Name` | _string_ | 存储桶名称 | +|`bucket.CreationDate` | _time.Time_ | 存储桶的创建时间 | + + +__示例__ + + +```go +buckets, err := minioClient.ListBuckets() +if err != nil { + fmt.Println(err) + return +} +for _, bucket := range buckets { + fmt.Println(bucket) +} +``` + + +### BucketExists(bucketName string) (found bool, err error) +检查存储桶是否存在。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | + + +__返回值__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`found` | _bool_ | 存储桶是否存在 | +|`err` | _error_ | 标准Error | + + +__示例__ + + +```go +found, err := minioClient.BucketExists("mybucket") +if err != nil { + fmt.Println(err) + return +} +if found { + fmt.Println("Bucket found") +} +``` + + +### RemoveBucket(bucketName string) error +删除一个存储桶，存储桶必须为空才能被成功删除。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | + +__示例__ + + +```go +err = minioClient.RemoveBucket("mybucket") +if err != nil { + fmt.Println(err) + return +} +``` + + +### ListObjects(bucketName, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo +列举存储桶里的对象。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectPrefix` |_string_ | 要列举的对象前缀 | +|`recursive` | _bool_ |`true`代表递归查找，`false`代表类似文件夹查找，以'/'分隔，不查子文件夹。 | +|`doneCh` | _chan struct{}_ | 在该channel上结束ListObjects iterator的一个message。 | + + +__返回值__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`objectInfo` | _chan minio.ObjectInfo_ |存储桶中所有对象的read channel，对象的格式如下： | + +__minio.ObjectInfo__ + +|属性 |类型 |描述 | +|:---|:---| :---| +|`objectInfo.Key` | _string_ |对象的名称 | +|`objectInfo.Size` | _int64_ |对象的大小 | +|`objectInfo.ETag` | _string_ |对象的MD5校验码 | +|`objectInfo.LastModified` | _time.Time_ |对象的最后修改时间 | + + +```go +// Create a done channel to control 'ListObjects' go routine. +doneCh := make(chan struct{}) + +// Indicate to our routine to exit cleanly upon return. +defer close(doneCh) + +isRecursive := true +objectCh := minioClient.ListObjects("mybucket", "myprefix", isRecursive, doneCh) +for object := range objectCh { + if object.Err != nil { + fmt.Println(object.Err) + return + } + fmt.Println(object) +} +``` + + + +### ListObjectsV2(bucketName, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo +使用listing API v2版本列举存储桶中的对象。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +| `objectPrefix` |_string_ | 要列举的对象前缀 | +| `recursive` | _bool_ |`true`代表递归查找，`false`代表类似文件夹查找，以'/'分隔，不查子文件夹。 | +|`doneCh` | _chan struct{}_ | 在该channel上结束ListObjects iterator的一个message。 | + + +__返回值__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`objectInfo` | _chan minio.ObjectInfo_ |存储桶中所有对象的read channel | + + +```go +// Create a done channel to control 'ListObjectsV2' go routine. +doneCh := make(chan struct{}) + +// Indicate to our routine to exit cleanly upon return. +defer close(doneCh) + +isRecursive := true +objectCh := minioClient.ListObjectsV2("mybucket", "myprefix", isRecursive, doneCh) +for object := range objectCh { + if object.Err != nil { + fmt.Println(object.Err) + return + } + fmt.Println(object) +} +``` + + +### ListIncompleteUploads(bucketName, prefix string, recursive bool, doneCh chan struct{}) <- chan ObjectMultipartInfo +列举存储桶中未完整上传的对象。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +| `prefix` |_string_ | 不完整上传的对象的前缀 | +| `recursive` | _bool_ |`true`代表递归查找，`false`代表类似文件夹查找，以'/'分隔，不查子文件夹。 | +|`doneCh` | _chan struct{}_ | 在该channel上结束ListIncompleteUploads iterator的一个message。 | + + +__返回值__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`multiPartInfo` | _chan minio.ObjectMultipartInfo_ |multipart对象格式如下： | + +__minio.ObjectMultipartInfo__ + +|属性 |类型 |描述 | +|:---|:---| :---| +|`multiPartObjInfo.Key` | _string_ |未完整上传的对象的名称 | +|`multiPartObjInfo.UploadID` | _string_ |未完整上传的对象的Upload ID | +|`multiPartObjInfo.Size` | _int64_ |未完整上传的对象的大小 | + +__示例__ + + +```go +// Create a done channel to control 'ListObjects' go routine. +doneCh := make(chan struct{}) + +// Indicate to our routine to exit cleanly upon return. +defer close(doneCh) + +isRecursive := true // Recursively list everything at 'myprefix' +multiPartObjectCh := minioClient.ListIncompleteUploads("mybucket", "myprefix", isRecursive, doneCh) +for multiPartObject := range multiPartObjectCh { + if multiPartObject.Err != nil { + fmt.Println(multiPartObject.Err) + return + } + fmt.Println(multiPartObject) +} +``` + +## 3. 操作对象 + + +### GetObject(bucketName, objectName string, opts GetObjectOptions) (*Object, error) +返回对象数据的流，error是读流时经常抛的那些错。 + + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`opts` | _minio.GetObjectOptions_ | GET请求的一些额外参数，像encryption，If-Match | + + +__minio.GetObjectOptions__ + +|参数 | 类型 | 描述 | +|:---|:---|:---| +| `opts.Materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go) | + +__返回值__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`object` | _*minio.Object_ |_minio.Object_代表了一个object reader。它实现了io.Reader, io.Seeker, io.ReaderAt and io.Closer接口。 | + + +__示例__ + + +```go +object, err := minioClient.GetObject("mybucket", "myobject", minio.GetObjectOptions{}) +if err != nil { + fmt.Println(err) + return +} +localFile, err := os.Create("/tmp/local-file.jpg") +if err != nil { + fmt.Println(err) + return +} +if _, err = io.Copy(localFile, object); err != nil { + fmt.Println(err) + return +} +``` + + +### FGetObject(bucketName, objectName, filePath string, opts GetObjectOptions) error +下载并将文件保存到本地文件系统。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`filePath` | _string_ |下载后保存的路径 | +|`opts` | _minio.GetObjectOptions_ | GET请求的一些额外参数，像encryption，If-Match | + + +__示例__ + + +```go +err = minioClient.FGetObject("mybucket", "myobject", "/tmp/myobject", minio.GetObjectOptions{}) +if err != nil { + fmt.Println(err) + return +} +``` + +### GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) +和GetObject操作是一样的，不过传入了取消请求的context。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`ctx` | _context.Context_ |请求上下文（Request context） | +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`opts` | _minio.GetObjectOptions_ | GET请求的一些额外参数，像encryption，If-Match | + + +__返回值__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`object` | _*minio.Object_ |_minio.Object_代表了一个object reader。它实现了io.Reader, io.Seeker, io.ReaderAt and io.Closer接口。 | + + +__示例__ + + +```go +ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) +defer cancel() + +object, err := minioClient.GetObjectWithContext(ctx, "mybucket", "myobject", minio.GetObjectOptions{}) +if err != nil { + fmt.Println(err) + return +} + +localFile, err := os.Create("/tmp/local-file.jpg") +if err != nil { + fmt.Println(err) + return +} + +if _, err = io.Copy(localFile, object); err != nil { + fmt.Println(err) + return +} +``` + + +### FGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error +和FGetObject操作是一样的，不过允许取消请求。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`ctx` | _context.Context_ |请求上下文 | +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`filePath` | _string_ |下载后保存的路径 | +|`opts` | _minio.GetObjectOptions_ | GET请求的一些额外参数，像encryption，If-Match | + + +__示例__ + + +```go +ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) +defer cancel() + +err = minioClient.FGetObjectWithContext(ctx, "mybucket", "myobject", "/tmp/myobject", minio.GetObjectOptions{}) +if err != nil { + fmt.Println(err) + return +} +``` + + +### FGetEncryptedObject(bucketName, objectName, filePath string, materials encrypt.Materials) error +和FGetObject操作是一样的，不过会对加密请求进行解密。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`filePath` | _string_ |下载后保存的路径| +|`materials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go) | + + +__示例__ + + +```go +// Generate a master symmetric key +key := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) + +// Build the CBC encryption material +cbcMaterials, err := encrypt.NewCBCSecureMaterials(key) +if err != nil { + fmt.Println(err) + return +} + +err = minioClient.FGetEncryptedObject("mybucket", "myobject", "/tmp/myobject", cbcMaterials) +if err != nil { + fmt.Println(err) + return +} +``` + + +### PutObject(bucketName, objectName string, reader io.Reader, objectSize int64,opts PutObjectOptions) (n int, err error) +当对象小于64MiB时，直接在一次PUT请求里进行上传。当大于64MiB时，根据文件的实际大小，PutObject会自动地将对象进行拆分成64MiB一块或更大一些进行上传。对象的最大大小是5TB。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`reader` | _io.Reader_ |任意实现了io.Reader的GO类型 | +|`objectSize`| _int64_ |上传的对象的大小，-1代表未知。 | +|`opts` | _minio.PutObjectOptions_ | 允许用户设置可选的自定义元数据，内容标题，加密密钥和用于分段上传操作的线程数量。 | + +__minio.PutObjectOptions__ + +|属性 | 类型 | 描述 | +|:--- |:--- | :--- | +| `opts.UserMetadata` | _map[string]string_ | 用户元数据的Map| +| `opts.Progress` | _io.Reader_ | 获取上传进度的Reader | +| `opts.ContentType` | _string_ | 对象的Content type， 例如"application/text" | +| `opts.ContentEncoding` | _string_ | 对象的Content encoding，例如"gzip" | +| `opts.ContentDisposition` | _string_ | 对象的Content disposition, "inline" | +| `opts.CacheControl` | _string_ | 指定针对请求和响应的缓存机制，例如"max-age=600"| +| `opts.EncryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go) | + + +__示例__ + + +```go +file, err := os.Open("my-testfile") +if err != nil { + fmt.Println(err) + return +} +defer file.Close() + +fileStat, err := file.Stat() +if err != nil { + fmt.Println(err) + return +} + +n, err := minioClient.PutObject("mybucket", "myobject", file, fileStat.Size(), minio.PutObjectOptions{ContentType:"application/octet-stream"}) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully uploaded bytes: ", n) +``` + +API方法在minio-go SDK版本v3.0.3中提供的PutObjectWithSize，PutObjectWithMetadata，PutObjectStreaming和PutObjectWithProgress被替换为接受指向PutObjectOptions struct的指针的新的PutObject调用变体。 + + +### PutObjectWithContext(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, opts PutObjectOptions) (n int, err error) +和PutObject是一样的，不过允许取消请求。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`ctx` | _context.Context_ |请求上下文 | +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`reader` | _io.Reader_ |任何实现io.Reader的Go类型 | +|`objectSize`| _int64_ | 上传的对象的大小，-1代表未知 | +|`opts` | _minio.PutObjectOptions_ |允许用户设置可选的自定义元数据，content-type，content-encoding，content-disposition以及cache-control headers，传递加密模块以加密对象，并可选地设置multipart put操作的线程数量。| + + +__示例__ + + +```go +ctx, cancel := context.WithTimeout(context.Background(), 10 * time.Second) +defer cancel() + +file, err := os.Open("my-testfile") +if err != nil { + fmt.Println(err) + return +} +defer file.Close() + +fileStat, err := file.Stat() +if err != nil { + fmt.Println(err) + return +} + +n, err := minioClient.PutObjectWithContext(ctx, "my-bucketname", "my-objectname", file, fileStat.Size(), minio.PutObjectOptions{ + ContentType: "application/octet-stream", +}) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully uploaded bytes: ", n) +``` + + +### CopyObject(dst DestinationInfo, src SourceInfo) error +通过在服务端对已存在的对象进行拷贝，实现新建或者替换对象。它支持有条件的拷贝，拷贝对象的一部分，以及在服务端的加解密。请查看`SourceInfo`和`DestinationInfo`两个类型来了解更多细节。 + +拷贝多个源文件到一个目标对象，请查看`ComposeObject` API。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`dst` | _minio.DestinationInfo_ |目标对象 | +|`src` | _minio.SourceInfo_ |源对象 | + + +__示例__ + + +```go +// Use-case 1: Simple copy object with no conditions. +// Source object +src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) + +// Destination object +dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + +```go +// Use-case 2: +// Copy object with copy-conditions, and copying only part of the source object. +// 1. that matches a given ETag +// 2. and modified after 1st April 2014 +// 3. but unmodified since 23rd April 2014 +// 4. copy only first 1MiB of object. + +// Source object +src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) + +// Set matching ETag condition, copy object which matches the following ETag. +src.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") + +// Set modified condition, copy object modified since 2014 April 1. +src.SetModifiedSinceCond(time.Date(2014, time.April, 1, 0, 0, 0, 0, time.UTC)) + +// Set unmodified condition, copy object unmodified since 2014 April 23. +src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 23, 0, 0, 0, 0, time.UTC)) + +// Set copy-range of only first 1MiB of file. +src.SetRange(0, 1024*1024-1) + +// Destination object +dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + + +### ComposeObject(dst minio.DestinationInfo, srcs []minio.SourceInfo) error +通过使用服务端拷贝实现钭多个源对象合并创建成一个新的对象。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---|:---| +|`dst` | _minio.DestinationInfo_ |要被创建的目标对象 | +|`srcs` | _[]minio.SourceInfo_ |要合并的多个源对象 | + + +__示例__ + + +```go +// Prepare source decryption key (here we assume same key to +// decrypt all source objects.) +decKey := minio.NewSSEInfo([]byte{1, 2, 3}, "") + +// Source objects to concatenate. We also specify decryption +// key for each +src1 := minio.NewSourceInfo("bucket1", "object1", &decKey) +src1.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") + +src2 := minio.NewSourceInfo("bucket2", "object2", &decKey) +src2.SetMatchETagCond("f8ef9c385918b653a31624deb84149d2") + +src3 := minio.NewSourceInfo("bucket3", "object3", &decKey) +src3.SetMatchETagCond("5918b653a31624deb84149d2f8ef9c38") + +// Create slice of sources. +srcs := []minio.SourceInfo{src1, src2, src3} + +// Prepare destination encryption key +encKey := minio.NewSSEInfo([]byte{8, 9, 0}, "") + +// Create destination info +dst, err := minio.NewDestinationInfo("bucket", "object", &encKey, nil) +if err != nil { + fmt.Println(err) + return +} + +// Compose object call by concatenating multiple source files. +err = minioClient.ComposeObject(dst, srcs) +if err != nil { + fmt.Println(err) + return +} + +fmt.Println("Composed object successfully.") +``` + + +### NewSourceInfo(bucket, object string, decryptSSEC *SSEInfo) SourceInfo +构建一个可用于服务端拷贝操作（像`CopyObject`和`ComposeObject`）的`SourceInfo`对象。该对象可用于给源对象设置拷贝条件。 + +__参数__ + +| 参数 | 类型 | 描述 | +| :--- | :--- | :--- | +| `bucket` | _string_ | 源存储桶 | +| `object` | _string_ | 源对象 | +| `decryptSSEC` | _*minio.SSEInfo_ | 源对象的解密信息 (`nil`代表不用解密) | + +__示例__ + +```go +// No decryption parameter. +src := minio.NewSourceInfo("bucket", "object", nil) + +// Destination object +dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + +```go +// With decryption parameter. +decKey := minio.NewSSEInfo([]byte{1,2,3}, "") +src := minio.NewSourceInfo("bucket", "object", &decKey) + +// Destination object +dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + + +### NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, userMeta map[string]string) (DestinationInfo, error) +构建一个用于服务端拷贝操作（像`CopyObject`和`ComposeObject`）的用作目标对象的`DestinationInfo`。 + +__参数__ + +| 参数 | 类型 | 描述 | +| :--- | :--- | :--- | +| `bucket` | _string_ | 目标存储桶名称 | +| `object` | _string_ | 目标对象名称 | +| `encryptSSEC` | _*minio.SSEInfo_ | 源对象的加密信息 (`nil`代表不用加密) | +| `userMeta` | _map[string]string_ | 给目标对象的用户元数据，如果是nil,并只有一个源对象，则将源对象的用户元数据拷贝给目标对象。| + +__示例__ + +```go +// No encryption parameter. +src := minio.NewSourceInfo("bucket", "object", nil) +dst, err := minio.NewDestinationInfo("bucket", "object", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + +```go +src := minio.NewSourceInfo("bucket", "object", nil) + +// With encryption parameter. +encKey := minio.NewSSEInfo([]byte{1,2,3}, "") +dst, err := minio.NewDestinationInfo("bucket", "object", &encKey, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + + +### FPutObject(bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) +将filePath对应的文件内容上传到一个对象中。 + +当对象小于64MiB时，FPutObject直接在一次PUT请求里进行上传。当大于64MiB时，根据文件的实际大小，FPutObject会自动地将对象进行拆分成64MiB一块或更大一些进行上传。对象的最大大小是5TB。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`filePath` | _string_ |要上传的文件的路径 | +|`opts` | _minio.PutObjectOptions_ |允许用户设置可选的自定义元数据，content-type，content-encoding，content-disposition以及cache-control headers，传递加密模块以加密对象，并可选地设置multipart put操作的线程数量。 | + + +__示例__ + + +```go +n, err := minioClient.FPutObject("my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ + ContentType: "application/csv", +}); +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully uploaded bytes: ", n) +``` + + +### FPutObjectWithContext(ctx context.Context, bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) +和FPutObject操作是一样的，不过允许取消请求。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`ctx` | _context.Context_ |请求上下文 | +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`filePath` | _string_ |要上传的文件的路径 | +|`opts` | _minio.PutObjectOptions_ |允许用户设置可选的自定义元数据，content-type，content-encoding，content-disposition以及cache-control headers，传递加密模块以加密对象，并可选地设置multipart put操作的线程数量。 | + +__示例__ + + +```go +ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) +defer cancel() + +n, err := minioClient.FPutObjectWithContext(ctx, "mybucket", "myobject.csv", "/tmp/otherobject.csv", minio.PutObjectOptions{ContentType:"application/csv"}) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully uploaded bytes: ", n) +``` + + +### StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) +获取对象的元数据。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`opts` | _minio.StatObjectOptions_ | GET info/stat请求的一些额外参数，像encryption，If-Match | + + +__返回值__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`objInfo` | _minio.ObjectInfo_ |对象stat信息 | + + +__minio.ObjectInfo__ + +|属性 |类型 |描述 | +|:---|:---| :---| +|`objInfo.LastModified` | _time.Time_ |对象的最后修改时间 | +|`objInfo.ETag` | _string_ |对象的MD5校验码| +|`objInfo.ContentType` | _string_ |对象的Content type| +|`objInfo.Size` | _int64_ |对象的大小| + + +__示例__ + + +```go +objInfo, err := minioClient.StatObject("mybucket", "myobject", minio.StatObjectOptions{}) +if err != nil { + fmt.Println(err) + return +} +fmt.Println(objInfo) +``` + + +### RemoveObject(bucketName, objectName string) error +删除一个对象。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | + + +```go +err = minioClient.RemoveObject("mybucket", "myobject") +if err != nil { + fmt.Println(err) + return +} +``` + + +### RemoveObjects(bucketName string, objectsCh chan string) (errorCh <-chan RemoveObjectError) + +从一个input channel里删除一个对象集合。一次发送到服务端的删除请求最多可删除1000个对象。通过error channel返回的错误信息。 + +__参数__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectsCh` | _chan string_ | 要删除的对象的channel | + + +__返回值__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`errorCh` | _<-chan minio.RemoveObjectError_ | 删除时观察到的错误的Receive-only channel。 | + + +```go +objectsCh := make(chan string) + +// Send object names that are needed to be removed to objectsCh +go func() { + defer close(objectsCh) + // List all objects from a bucket-name with a matching prefix. + for object := range minioClient.ListObjects("my-bucketname", "my-prefixname", true, nil) { + if object.Err != nil { + log.Fatalln(object.Err) + } + objectsCh <- object.Key + } +}() + +for rErr := range minioClient.RemoveObjects("mybucket", objectsCh) { + fmt.Println("Error detected during deletion: ", rErr) +} +``` + + +### RemoveIncompleteUpload(bucketName, objectName string) error +删除一个未完整上传的对象。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | + +__示例__ + + +```go +err = minioClient.RemoveIncompleteUpload("mybucket", "myobject") +if err != nil { + fmt.Println(err) + return +} +``` + +## 4. 操作加密对象 + + +### NewSymmetricKey(key []byte) *encrypt.SymmetricKey + +__参数__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`key` | _string_ |存储桶名称 | + + +__返回值__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`symmetricKey` | _*encrypt.SymmetricKey_ | 加密解密的对称秘钥 | + +```go +symKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) + +// Build the CBC encryption material with symmetric key. +cbcMaterials, err := encrypt.NewCBCSecureMaterials(symKey) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully initialized Symmetric key CBC materials", cbcMaterials) + +object, err := minioClient.GetEncryptedObject("mybucket", "myobject", cbcMaterials) +if err != nil { + fmt.Println(err) + return +} +defer object.Close() +``` + + +### NewAsymmetricKey(privateKey []byte, publicKey[]byte) (*encrypt.AsymmetricKey, error) + +__参数__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`privateKey` | _[]byte_ | Private key数据 | +|`publicKey` | _[]byte_ | Public key数据 | + + +__返回值__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`asymmetricKey` | _*encrypt.AsymmetricKey_ | 加密解密的非对称秘钥 | +|`err` | _error_ | 标准Error | + + +```go +privateKey, err := ioutil.ReadFile("private.key") +if err != nil { + fmt.Println(err) + return +} + +publicKey, err := ioutil.ReadFile("public.key") +if err != nil { + fmt.Println(err) + return +} + +// Initialize the asymmetric key +asymmetricKey, err := encrypt.NewAsymmetricKey(privateKey, publicKey) +if err != nil { + fmt.Println(err) + return +} + +// Build the CBC encryption material for asymmetric key. +cbcMaterials, err := encrypt.NewCBCSecureMaterials(asymmetricKey) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully initialized Asymmetric key CBC materials", cbcMaterials) + +object, err := minioClient.GetEncryptedObject("mybucket", "myobject", cbcMaterials) +if err != nil { + fmt.Println(err) + return +} +defer object.Close() +``` + + +### GetEncryptedObject(bucketName, objectName string, encryptMaterials encrypt.Materials) (io.ReadCloser, error) + +返回对象的解密流。读流时的常见错误。 + +__参数__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ | 存储桶名称 | +|`objectName` | _string_ | 对象的名称 | +|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go) | + + +__返回值__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`stream` | _io.ReadCloser_ | 返回对象的reader,调用者需要在读取之后进行关闭。 | +|`err` | _error | 错误信息 | + + +__示例__ + + +```go +// Generate a master symmetric key +key := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) + +// Build the CBC encryption material +cbcMaterials, err := encrypt.NewCBCSecureMaterials(key) +if err != nil { + fmt.Println(err) + return +} + +object, err := minioClient.GetEncryptedObject("mybucket", "myobject", cbcMaterials) +if err != nil { + fmt.Println(err) + return +} +defer object.Close() + +localFile, err := os.Create("/tmp/local-file.jpg") +if err != nil { + fmt.Println(err) + return +} +defer localFile.Close() + +if _, err = io.Copy(localFile, object); err != nil { + fmt.Println(err) + return +} +``` + + + +### PutEncryptedObject(bucketName, objectName string, reader io.Reader, encryptMaterials encrypt.Materials) (n int, err error) +加密并上传对象。 + +__参数__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`reader` | _io.Reader_ |任何实现io.Reader的Go类型 | +|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go) | + +__示例__ + +```go +// Load a private key +privateKey, err := ioutil.ReadFile("private.key") +if err != nil { + fmt.Println(err) + return +} + +// Load a public key +publicKey, err := ioutil.ReadFile("public.key") +if err != nil { + fmt.Println(err) + return +} + +// Build an asymmetric key +key, err := encrypt.NewAsymmetricKey(privateKey, publicKey) +if err != nil { + fmt.Println(err) + return +} + +// Build the CBC encryption module +cbcMaterials, err := encrypt.NewCBCSecureMaterials(key) +if err != nil { + fmt.Println(err) + return +} + +// Open a file to upload +file, err := os.Open("my-testfile") +if err != nil { + fmt.Println(err) + return +} +defer file.Close() + +// Upload the encrypted form of the file +n, err := minioClient.PutEncryptedObject("mybucket", "myobject", file, cbcMaterials) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully uploaded encrypted bytes: ", n) +``` + + +### FPutEncryptedObject(bucketName, objectName, filePath, encryptMaterials encrypt.Materials) (n int, err error) +通过一个文件进行加密并上传到对象。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`filePath` | _string_ |要上传的文件的路径 | +|`encryptMaterials` | _encrypt.Materials_ | `encrypt`包提供的对流加密的接口，(更多信息，请看https://godoc.org/github.com/minio/minio-go) | + +__示例__ + + +```go +// Load a private key +privateKey, err := ioutil.ReadFile("private.key") +if err != nil { + fmt.Println(err) + return +} + +// Load a public key +publicKey, err := ioutil.ReadFile("public.key") +if err != nil { + fmt.Println(err) + return +} + +// Build an asymmetric key +key, err := encrypt.NewAsymmetricKey(privateKey, publicKey) +if err != nil { + fmt.Println(err) + return +} + +// Build the CBC encryption module +cbcMaterials, err := encrypt.NewCBCSecureMaterials(key) +if err != nil { + fmt.Println(err) + return +} + +n, err := minioClient.FPutEncryptedObject("mybucket", "myobject.csv", "/tmp/otherobject.csv", cbcMaterials) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully uploaded encrypted bytes: ", n) +``` + + + +### NewSSEInfo(key []byte, algo string) SSEInfo +创建一个通过用户提供的key(SSE-C),进行服务端加解密操作的key对象。 + +__参数__ + +| 参数 | 类型 | 描述 | +| :--- | :--- | :--- | +| `key` | _[]byte_ | 未编码的二进制key数组 | +| `algo` | _string_ | 加密算法，可以为空（默认是`AES256`） | + + +## 5. Presigned操作 + + +### PresignedGetObject(bucketName, objectName string, expiry time.Duration, reqParams url.Values) (*url.URL, error) +生成一个用于HTTP GET操作的presigned URL。浏览器/移动客户端可以在即使存储桶为私有的情况下也可以通过这个URL进行下载。这个presigned URL可以有一个过期时间，默认是7天。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`expiry` | _time.Duration_ |presigned URL的过期时间，单位是秒 | +|`reqParams` | _url.Values_ |额外的响应头，支持_response-expires_， _response-content-type_， _response-cache-control_， _response-content-disposition_。 | + + +__示例__ + + +```go +// Set request parameters for content-disposition. +reqParams := make(url.Values) +reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") + +// Generates a presigned url which expires in a day. +presignedURL, err := minioClient.PresignedGetObject("mybucket", "myobject", time.Second * 24 * 60 * 60, reqParams) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully generated presigned URL", presignedURL) +``` + + +### PresignedPutObject(bucketName, objectName string, expiry time.Duration) (*url.URL, error) +生成一个用于HTTP GET操作的presigned URL。浏览器/移动客户端可以在即使存储桶为私有的情况下也可以通过这个URL进行下载。这个presigned URL可以有一个过期时间，默认是7天。 + +注意：你可以通过只指定对象名称上传到S3。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`expiry` | _time.Duration_ |presigned URL的过期时间，单位是秒 | + + +__示例__ + + +```go +// Generates a url which expires in a day. +expiry := time.Second * 24 * 60 * 60 // 1 day. +presignedURL, err := minioClient.PresignedPutObject("mybucket", "myobject", expiry) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully generated presigned URL", presignedURL) +``` + + +### PresignedHeadObject(bucketName, objectName string, expiry time.Duration, reqParams url.Values) (*url.URL, error) +生成一个用于HTTP GET操作的presigned URL。浏览器/移动客户端可以在即使存储桶为私有的情况下也可以通过这个URL进行下载。这个presigned URL可以有一个过期时间，默认是7天。 + +__参数__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectName` | _string_ |对象的名称 | +|`expiry` | _time.Duration_ |presigned URL的过期时间，单位是秒 | +|`reqParams` | _url.Values_ |额外的响应头，支持_response-expires_， _response-content-type_， _response-cache-control_， _response-content-disposition_。 | + + +__示例__ + + +```go +// Set request parameters for content-disposition. +reqParams := make(url.Values) +reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") + +// Generates a presigned url which expires in a day. +presignedURL, err := minioClient.PresignedHeadObject("mybucket", "myobject", time.Second * 24 * 60 * 60, reqParams) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully generated presigned URL", presignedURL) +``` + + +### PresignedPostPolicy(PostPolicy) (*url.URL, map[string]string, error) +允许给POST操作的presigned URL设置策略条件。这些策略包括比如，接收对象上传的存储桶名称，名称前缀，过期策略。 + +```go +// Initialize policy condition config. +policy := minio.NewPostPolicy() + +// Apply upload policy restrictions: +policy.SetBucket("mybucket") +policy.SetKey("myobject") +policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days + +// Only allow 'png' images. +policy.SetContentType("image/png") + +// Only allow content size in range 1KB to 1MB. +policy.SetContentLengthRange(1024, 1024*1024) + +// Add a user metadata using the key "custom" and value "user" +policy.SetUserMetadata("custom", "user") + +// Get the POST form key/value object: +url, formData, err := minioClient.PresignedPostPolicy(policy) +if err != nil { + fmt.Println(err) + return +} + +// POST your content from the command line using `curl` +fmt.Printf("curl ") +for k, v := range formData { + fmt.Printf("-F %s=%s ", k, v) +} +fmt.Printf("-F file=@/etc/bash.bashrc ") +fmt.Printf("%s\n", url) +``` + +## 6. 存储桶策略/通知 + + +### SetBucketPolicy(bucketname, objectPrefix string, policy policy.BucketPolicy) error +给存储桶或者对象前缀设置访问权限。 + +必须引入`github.com/minio/minio-go/pkg/policy`包。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称| +|`objectPrefix` | _string_ |对象的名称前缀| +|`policy` | _policy.BucketPolicy_ |Policy的取值如下： | +| | | _policy.BucketPolicyNone_ | +| | | _policy.BucketPolicyReadOnly_ | +| | | _policy.BucketPolicyReadWrite_ | +| | | _policy.BucketPolicyWriteOnly_ | + + +__返回值__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`err` | _error_ |标准Error | + + +__示例__ + + +```go +// Sets 'mybucket' with a sub-directory 'myprefix' to be anonymously accessible for +// both read and write operations. +err = minioClient.SetBucketPolicy("mybucket", "myprefix", policy.BucketPolicyReadWrite) +if err != nil { + fmt.Println(err) + return +} +``` + + +### GetBucketPolicy(bucketName, objectPrefix string) (policy.BucketPolicy, error) +获取存储桶或者对象前缀的访问权限。 + +必须引入`github.com/minio/minio-go/pkg/policy`包。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`objectPrefix` | _string_ |该存储桶下的对象前缀 | + +__返回值__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketPolicy` | _policy.BucketPolicy_ |取值如下： `none`, `readonly`, `readwrite`,或者`writeonly` | +|`err` | _error_ |标准Error | + +__示例__ + + +```go +bucketPolicy, err := minioClient.GetBucketPolicy("mybucket", "") +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Access permissions for mybucket is", bucketPolicy) +``` + + +### GetBucketNotification(bucketName string) (BucketNotification, error) +获取存储桶的通知配置 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | + +__返回值__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketNotification` | _minio.BucketNotification_ |含有所有通知配置的数据结构| +|`err` | _error_ |标准Error | + +__示例__ + + +```go +bucketNotification, err := minioClient.GetBucketNotification("mybucket") +if err != nil { + fmt.Println("Failed to get bucket notification configurations for mybucket", err) + return +} + +for _, queueConfig := range bucketNotification.QueueConfigs { + for _, e := range queueConfig.Events { + fmt.Println(e + " event is enabled") + } +} +``` + + +### SetBucketNotification(bucketName string, bucketNotification BucketNotification) error +给存储桶设置新的通知 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | +|`bucketNotification` | _minio.BucketNotification_ |发送给配置的web service的XML | + +__返回值__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`err` | _error_ |标准Error | + +__示例__ + + +```go +queueArn := minio.NewArn("aws", "sqs", "us-east-1", "804605494417", "PhotoUpdate") + +queueConfig := minio.NewNotificationConfig(queueArn) +queueConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) +queueConfig.AddFilterPrefix("photos/") +queueConfig.AddFilterSuffix(".jpg") + +bucketNotification := minio.BucketNotification{} +bucketNotification.AddQueue(queueConfig) + +err = minioClient.SetBucketNotification("mybucket", bucketNotification) +if err != nil { + fmt.Println("Unable to set the bucket notification: ", err) + return +} +``` + + +### RemoveAllBucketNotification(bucketName string) error +删除存储桶上所有配置的通知 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ |存储桶名称 | + +__返回值__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`err` | _error_ |标准Error | + +__示例__ + + +```go +err = minioClient.RemoveAllBucketNotification("mybucket") +if err != nil { + fmt.Println("Unable to remove bucket notifications.", err) + return +} +``` + + +### ListenBucketNotification(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo +ListenBucketNotification API通过notification channel接收存储桶通知事件。返回的notification channel有两个属性，'Records'和'Err'。 + +- 'Records'持有从服务器返回的通知信息。 +- 'Err'表示的是处理接收到的通知时报的任何错误。 + +注意：一旦报错，notification channel就会关闭。 + +__参数__ + + +|参数 |类型 |描述 | +|:---|:---| :---| +|`bucketName` | _string_ | 被监听通知的存储桶 | +|`prefix` | _string_ | 过滤通知的对象前缀 | +|`suffix` | _string_ | 过滤通知的对象后缀 | +|`events` | _[]string_ | 开启指定事件类型的通知 | +|`doneCh` | _chan struct{}_ | 在该channel上结束ListenBucketNotification iterator的一个message。 | + +__返回值__ + +|参数 |类型 |描述 | +|:---|:---| :---| +|`notificationInfo` | _chan minio.NotificationInfo_ | 存储桶通知的channel | + +__minio.NotificationInfo__ + +|属性 |类型 |描述 | +|`notificationInfo.Records` | _[]minio.NotificationEvent_ | 通知事件的集合 | +|`notificationInfo.Err` | _error_ | 操作时报的任何错误(标准Error) | + + +__示例__ + + +```go +// Create a done channel to control 'ListenBucketNotification' go routine. +doneCh := make(chan struct{}) + +// Indicate a background go-routine to exit cleanly upon return. +defer close(doneCh) + +// Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. +for notificationInfo := range minioClient.ListenBucketNotification("mybucket", "myprefix/", ".mysuffix", []string{ + "s3:ObjectCreated:*", + "s3:ObjectAccessed:*", + "s3:ObjectRemoved:*", + }, doneCh) { + if notificationInfo.Err != nil { + fmt.Println(notificationInfo.Err) + } + fmt.Println(notificationInfo) +} +``` + +## 7. 客户端自定义设置 + + +### SetAppInfo(appName, appVersion string) +给User-Agent添加的自定义应用信息。 + +__参数__ + +| 参数 | 类型 | 描述 | +|---|---|---| +|`appName` | _string_ | 发请求的应用名称 | +| `appVersion`| _string_ | 发请求的应用版本 | + + +__示例__ + + +```go +// Set Application name and version to be used in subsequent API requests. +minioClient.SetAppInfo("myCloudApp", "1.0.0") +``` + + +### SetCustomTransport(customHTTPTransport http.RoundTripper) +重写默认的HTTP transport，通常用于调试或者添加自定义的TLS证书。 + +__参数__ + +| 参数 | 类型 | 描述 | +|---|---|---| +|`customHTTPTransport` | _http.RoundTripper_ | 自定义的transport，例如：为了调试对API请求响应进行追踪。| + + + +### TraceOn(outputStream io.Writer) +开启HTTP tracing。追踪信息输出到io.Writer，如果outputstream为nil，则trace写入到os.Stdout标准输出。 + +__参数__ + +| 参数 | 类型 | 描述 | +|---|---|---| +|`outputStream` | _io.Writer_ | HTTP trace写入到outputStream | + + + +### TraceOff() +关闭HTTP tracing。 + + +### SetS3TransferAccelerate(acceleratedEndpoint string) +给后续所有API请求设置ASW S3传输加速endpoint。 +注意：此API仅对AWS S3有效，对其它S3兼容的对象存储服务不生效。 + +__参数__ + +| 参数 | 类型 | 描述 | +|---|---|---| +|`acceleratedEndpoint` | _string_ | 设置新的S3传输加速endpoint。| + + +## 8. 了解更多 + +- [用Go语言创建属于你的音乐播放器APP示例](https://docs.minio.io/docs/go-music-player-app) diff --git a/docs/zh_CN/CONTRIBUTING.md b/docs/zh_CN/CONTRIBUTING.md new file mode 100644 index 0000000..bc408c6 --- /dev/null +++ b/docs/zh_CN/CONTRIBUTING.md @@ -0,0 +1,22 @@ + +### 开发者指南 + +``minio-go``欢迎你的贡献。为了让大家配合更加默契，我们做出如下约定： + +* fork项目并修改，我们鼓励大家使用pull requests进行代码相关的讨论。 + - Fork项目 + - 创建你的特性分支 (git checkout -b my-new-feature) + - Commit你的修改(git commit -am 'Add some feature') + - Push到远程分支(git push origin my-new-feature) + - 创建一个Pull Request + +* 当你准备创建pull request时，请确保： + - 写单元测试，如果你有什么疑问，请在pull request中提出来。 + - 运行`go fmt` + - 将你的多个提交合并成一个提交： `git rebase -i`。你可以强制update你的pull request。 + - 确保`go test -race ./...`和`go build`完成。 + 注意：go test会进行功能测试，这需要你有一个AWS S3账号。将账户信息设为``ACCESS_KEY``和``SECRET_KEY``环境变量。如果想运行简版测试，请使用``go test -short -race ./...``。 + +* 请阅读 [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) + - `minio-go`项目严格符合Golang风格 + - 如果您看到代码有问题，请随时发一个pull request diff --git a/examples/minio/listenbucketnotification.go b/examples/minio/listenbucketnotification.go new file mode 100644 index 0000000..4c48510 --- /dev/null +++ b/examples/minio/listenbucketnotification.go @@ -0,0 +1,61 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + minioClient, err := minio.New("play.minio.io:9000", "YOUR-ACCESS", "YOUR-SECRET", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Create a done channel to control 'ListenBucketNotification' go routine. + doneCh := make(chan struct{}) + + // Indicate to our routine to exit cleanly upon return. + defer close(doneCh) + + // Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. + for notificationInfo := range minioClient.ListenBucketNotification("YOUR-BUCKET", "PREFIX", "SUFFIX", []string{ + "s3:ObjectCreated:*", + "s3:ObjectAccessed:*", + "s3:ObjectRemoved:*", + }, doneCh) { + if notificationInfo.Err != nil { + log.Fatalln(notificationInfo.Err) + } + log.Println(notificationInfo) + } +} diff --git a/examples/s3/bucketexists.go b/examples/s3/bucketexists.go new file mode 100644 index 0000000..20dea30 --- /dev/null +++ b/examples/s3/bucketexists.go @@ -0,0 +1,52 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + found, err := s3Client.BucketExists("my-bucketname") + if err != nil { + log.Fatalln(err) + } + + if found { + log.Println("Bucket found.") + } else { + log.Println("Bucket not found.") + } +} diff --git a/examples/s3/composeobject.go b/examples/s3/composeobject.go new file mode 100644 index 0000000..2f76ff0 --- /dev/null +++ b/examples/s3/composeobject.go @@ -0,0 +1,78 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + minio "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // Enable trace. + // s3Client.TraceOn(os.Stderr) + + // Prepare source decryption key (here we assume same key to + // decrypt all source objects.) + decKey := minio.NewSSEInfo([]byte{1, 2, 3}, "") + + // Source objects to concatenate. We also specify decryption + // key for each + src1 := minio.NewSourceInfo("bucket1", "object1", &decKey) + src1.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") + + src2 := minio.NewSourceInfo("bucket2", "object2", &decKey) + src2.SetMatchETagCond("f8ef9c385918b653a31624deb84149d2") + + src3 := minio.NewSourceInfo("bucket3", "object3", &decKey) + src3.SetMatchETagCond("5918b653a31624deb84149d2f8ef9c38") + + // Create slice of sources. + srcs := []minio.SourceInfo{src1, src2, src3} + + // Prepare destination encryption key + encKey := minio.NewSSEInfo([]byte{8, 9, 0}, "") + + // Create destination info + dst, err := minio.NewDestinationInfo("bucket", "object", &encKey, nil) + if err != nil { + log.Fatalln(err) + } + + err = s3Client.ComposeObject(dst, srcs) + if err != nil { + log.Fatalln(err) + } + + log.Println("Composed object successfully.") +} diff --git a/examples/s3/copyobject.go b/examples/s3/copyobject.go new file mode 100644 index 0000000..a7c3eca --- /dev/null +++ b/examples/s3/copyobject.go @@ -0,0 +1,75 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "time" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // Enable trace. + // s3Client.TraceOn(os.Stderr) + + // Source object + src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) + + // All following conditions are allowed and can be combined together. + + // Set modified condition, copy object modified since 2014 April. + src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + + // Set unmodified condition, copy object unmodified since 2014 April. + // src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + + // Set matching ETag condition, copy object which matches the following ETag. + // src.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") + + // Set matching ETag except condition, copy object which does not match the following ETag. + // src.SetMatchETagExceptCond("31624deb84149d2f8ef9c385918b653a") + + // Destination object + dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) + if err != nil { + log.Fatalln(err) + } + + // Initiate copy object. + err = s3Client.CopyObject(dst, src) + if err != nil { + log.Fatalln(err) + } + log.Println("Copied source object /my-sourcebucketname/my-sourceobjectname to destination /my-bucketname/my-objectname Successfully.") +} diff --git a/examples/s3/fgetobject-context.go b/examples/s3/fgetobject-context.go new file mode 100644 index 0000000..6004baa --- /dev/null +++ b/examples/s3/fgetobject-context.go @@ -0,0 +1,54 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "time" + + "context" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname + // and my-filename.csv are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + if err := s3Client.FGetObjectWithContext(ctx, "my-bucketname", "my-objectname", "my-filename.csv", minio.GetObjectOptions{}); err != nil { + log.Fatalln(err) + } + log.Println("Successfully saved my-filename.csv") + +} diff --git a/examples/s3/fgetobject.go b/examples/s3/fgetobject.go new file mode 100644 index 0000000..819a34f --- /dev/null +++ b/examples/s3/fgetobject.go @@ -0,0 +1,46 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname + // and my-filename.csv are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + if err := s3Client.FGetObject("my-bucketname", "my-objectname", "my-filename.csv", minio.GetObjectOptions{}); err != nil { + log.Fatalln(err) + } + log.Println("Successfully saved my-filename.csv") +} diff --git a/examples/s3/fputencrypted-object.go b/examples/s3/fputencrypted-object.go new file mode 100644 index 0000000..5da9f9d --- /dev/null +++ b/examples/s3/fputencrypted-object.go @@ -0,0 +1,59 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go/pkg/encrypt" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + filePath := "my-testfile" // Specify a local file that we will upload + bucketname := "my-bucketname" // Specify a bucket name - the bucket must already exist + objectName := "my-objectname" // Specify a object name + password := "correct horse battery staple" // Specify your password. DO NOT USE THIS ONE - USE YOUR OWN. + + // New SSE-C where the cryptographic key is derived from a password and the objectname + bucketname as salt + encryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketname+objectName)) + + // Encrypt file content and upload to the server + n, err := s3Client.FPutObject(bucketname, objectName, filePath, minio.PutObjectOptions{ServerSideEncryption: encryption}) + if err != nil { + log.Fatalln(err) + } + + log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") +} diff --git a/examples/s3/fputobject-context.go b/examples/s3/fputobject-context.go new file mode 100644 index 0000000..d7c941c --- /dev/null +++ b/examples/s3/fputobject-context.go @@ -0,0 +1,53 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "time" + + "context" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname + // and my-filename.csv are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + if _, err := s3Client.FPutObjectWithContext(ctx, "my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ContentType: "application/csv"}); err != nil { + log.Fatalln(err) + } + log.Println("Successfully uploaded my-filename.csv") +} diff --git a/examples/s3/fputobject.go b/examples/s3/fputobject.go new file mode 100644 index 0000000..34d8768 --- /dev/null +++ b/examples/s3/fputobject.go @@ -0,0 +1,48 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname + // and my-filename.csv are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + if _, err := s3Client.FPutObject("my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ + ContentType: "application/csv", + }); err != nil { + log.Fatalln(err) + } + log.Println("Successfully uploaded my-filename.csv") +} diff --git a/examples/s3/get-encrypted-object.go b/examples/s3/get-encrypted-object.go new file mode 100644 index 0000000..62a06d5 --- /dev/null +++ b/examples/s3/get-encrypted-object.go @@ -0,0 +1,69 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "io" + "log" + "os" + + "github.com/minio/minio-go" + "github.com/minio/minio-go/pkg/encrypt" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + + bucketname := "my-bucketname" // Specify a bucket name - the bucket must already exist + objectName := "my-objectname" // Specify a object name - the object must already exist + password := "correct horse battery staple" // Specify your password. DO NOT USE THIS ONE - USE YOUR OWN. + + // New SSE-C where the cryptographic key is derived from a password and the objectname + bucketname as salt + encryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketname+objectName)) + + // Get the encrypted object + reader, err := s3Client.GetObject(bucketname, objectName, minio.GetObjectOptions{ServerSideEncryption: encryption}) + if err != nil { + log.Fatalln(err) + } + defer reader.Close() + + // Local file which holds plain data + localFile, err := os.Create("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer localFile.Close() + + if _, err := io.Copy(localFile, reader); err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/getbucketlifecycle.go b/examples/s3/getbucketlifecycle.go new file mode 100644 index 0000000..2e3ef41 --- /dev/null +++ b/examples/s3/getbucketlifecycle.go @@ -0,0 +1,65 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "io" + "log" + "os" + "strings" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Get bucket lifecycle from S3 + lifecycle, err := s3Client.GetBucketLifecycle("my-bucketname") + if err != nil { + log.Fatalln(err) + } + + // Create lifecycle file + localLifecycleFile, err := os.Create("lifecycle.json") + if err != nil { + log.Fatalln(err) + } + defer localLifecycleFile.Close() + + lifecycleReader := strings.NewReader(lifecycle) + + if _, err := io.Copy(localLifecycleFile, lifecycleReader); err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/getbucketnotification.go b/examples/s3/getbucketnotification.go new file mode 100644 index 0000000..19349ba --- /dev/null +++ b/examples/s3/getbucketnotification.go @@ -0,0 +1,56 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + notifications, err := s3Client.GetBucketNotification("my-bucketname") + if err != nil { + log.Fatalln(err) + } + + log.Println("Bucket notification are successfully retrieved.") + + for _, topicConfig := range notifications.TopicConfigs { + for _, e := range topicConfig.Events { + log.Println(e + " event is enabled.") + } + } +} diff --git a/examples/s3/getbucketpolicy.go b/examples/s3/getbucketpolicy.go new file mode 100644 index 0000000..e5b5940 --- /dev/null +++ b/examples/s3/getbucketpolicy.go @@ -0,0 +1,64 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "io" + "log" + "os" + "strings" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + policy, err := s3Client.GetBucketPolicy("my-bucketname") + if err != nil { + log.Fatalln(err) + } + + // Create policy file + localFile, err := os.Create("policy.json") + if err != nil { + log.Fatalln(err) + } + defer localFile.Close() + + policyReader := strings.NewReader(policy) + + if _, err := io.Copy(localFile, policyReader); err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/getobject-client-encryption.go b/examples/s3/getobject-client-encryption.go new file mode 100644 index 0000000..6b06073 --- /dev/null +++ b/examples/s3/getobject-client-encryption.go @@ -0,0 +1,66 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "os" + "path" + + "github.com/minio/minio-go" + "github.com/minio/sio" + "golang.org/x/crypto/argon2" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + obj, err := s3Client.GetObject("my-bucketname", "my-objectname", minio.GetObjectOptions{}) + if err != nil { + log.Fatalln(err) + } + + localFile, err := os.Create("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer localFile.Close() + + password := []byte("myfavoritepassword") // Change as per your needs. + salt := []byte(path.Join("my-bucketname", "my-objectname")) // Change as per your needs. + _, err = sio.Decrypt(localFile, obj, sio.Config{ + Key: argon2.IDKey(password, salt, 1, 64*1024, 4, 32), // generate a 256 bit long key. + }) + if err != nil { + log.Fatalln(err) + } + log.Println("Successfully decrypted 'my-objectname' to local file 'my-testfile'") +} diff --git a/examples/s3/getobject-context.go b/examples/s3/getobject-context.go new file mode 100644 index 0000000..c7d4170 --- /dev/null +++ b/examples/s3/getobject-context.go @@ -0,0 +1,73 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "io" + "log" + "os" + "time" + + "context" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + opts := minio.GetObjectOptions{} + opts.SetModified(time.Now().Round(10 * time.Minute)) // get object if was modified within the last 10 minutes + reader, err := s3Client.GetObjectWithContext(ctx, "my-bucketname", "my-objectname", opts) + if err != nil { + log.Fatalln(err) + } + defer reader.Close() + + localFile, err := os.Create("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer localFile.Close() + + stat, err := reader.Stat() + if err != nil { + log.Fatalln(err) + } + + if _, err := io.CopyN(localFile, reader, stat.Size); err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/getobject.go b/examples/s3/getobject.go new file mode 100644 index 0000000..e17ef81 --- /dev/null +++ b/examples/s3/getobject.go @@ -0,0 +1,64 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "io" + "log" + "os" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + + reader, err := s3Client.GetObject("my-bucketname", "my-objectname", minio.GetObjectOptions{}) + if err != nil { + log.Fatalln(err) + } + defer reader.Close() + + localFile, err := os.Create("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer localFile.Close() + + stat, err := reader.Stat() + if err != nil { + log.Fatalln(err) + } + + if _, err := io.CopyN(localFile, reader, stat.Size); err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/getobjectacl.go b/examples/s3/getobjectacl.go new file mode 100644 index 0000000..f2bbd95 --- /dev/null +++ b/examples/s3/getobjectacl.go @@ -0,0 +1,53 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018-2019 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + + minio "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + + objectInfo, err := s3Client.GetObjectACL("my-bucketname", "my-objectname") + if err != nil { + log.Fatalln(err) + } + + //print all value header (acl, metadata, standard header value...) + for k, v := range objectInfo.Metadata { + fmt.Println("key:", k) + fmt.Printf(" - value: %v\n", v) + } +} diff --git a/examples/s3/listbuckets.go b/examples/s3/listbuckets.go new file mode 100644 index 0000000..5eae587 --- /dev/null +++ b/examples/s3/listbuckets.go @@ -0,0 +1,49 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID and YOUR-SECRETACCESSKEY are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + buckets, err := s3Client.ListBuckets() + if err != nil { + log.Fatalln(err) + } + for _, bucket := range buckets { + log.Println(bucket) + } +} diff --git a/examples/s3/listincompleteuploads.go b/examples/s3/listincompleteuploads.go new file mode 100644 index 0000000..a5a79b6 --- /dev/null +++ b/examples/s3/listincompleteuploads.go @@ -0,0 +1,58 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // Create a done channel to control 'ListObjects' go routine. + doneCh := make(chan struct{}) + + // Indicate to our routine to exit cleanly upon return. + defer close(doneCh) + + // List all multipart uploads from a bucket-name with a matching prefix. + for multipartObject := range s3Client.ListIncompleteUploads("my-bucketname", "my-prefixname", true, doneCh) { + if multipartObject.Err != nil { + fmt.Println(multipartObject.Err) + return + } + fmt.Println(multipartObject) + } + return +} diff --git a/examples/s3/listobjects-N.go b/examples/s3/listobjects-N.go new file mode 100644 index 0000000..55bceb4 --- /dev/null +++ b/examples/s3/listobjects-N.go @@ -0,0 +1,77 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + fmt.Println(err) + return + } + + // List 'N' number of objects from a bucket-name with a matching prefix. + listObjectsN := func(bucket, prefix string, recursive bool, N int) (objsInfo []minio.ObjectInfo, err error) { + // Create a done channel to control 'ListObjects' go routine. + doneCh := make(chan struct{}, 1) + + // Free the channel upon return. + defer close(doneCh) + + i := 1 + for object := range s3Client.ListObjects(bucket, prefix, recursive, doneCh) { + if object.Err != nil { + return nil, object.Err + } + i++ + // Verify if we have printed N objects. + if i == N { + // Indicate ListObjects go-routine to exit and stop + // feeding the objectInfo channel. + doneCh <- struct{}{} + } + objsInfo = append(objsInfo, object) + } + return objsInfo, nil + } + + // List recursively first 100 entries for prefix 'my-prefixname'. + recursive := true + objsInfo, err := listObjectsN("my-bucketname", "my-prefixname", recursive, 100) + if err != nil { + fmt.Println(err) + } + + // Print all the entries. + fmt.Println(objsInfo) +} diff --git a/examples/s3/listobjects.go b/examples/s3/listobjects.go new file mode 100644 index 0000000..1da2e3f --- /dev/null +++ b/examples/s3/listobjects.go @@ -0,0 +1,58 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + fmt.Println(err) + return + } + + // Create a done channel to control 'ListObjects' go routine. + doneCh := make(chan struct{}) + + // Indicate to our routine to exit cleanly upon return. + defer close(doneCh) + + // List all objects from a bucket-name with a matching prefix. + for object := range s3Client.ListObjects("my-bucketname", "my-prefixname", true, doneCh) { + if object.Err != nil { + fmt.Println(object.Err) + return + } + fmt.Println(object) + } + return +} diff --git a/examples/s3/listobjectsV2.go b/examples/s3/listobjectsV2.go new file mode 100644 index 0000000..190aec3 --- /dev/null +++ b/examples/s3/listobjectsV2.go @@ -0,0 +1,58 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + fmt.Println(err) + return + } + + // Create a done channel to control 'ListObjects' go routine. + doneCh := make(chan struct{}) + + // Indicate to our routine to exit cleanly upon return. + defer close(doneCh) + + // List all objects from a bucket-name with a matching prefix. + for object := range s3Client.ListObjectsV2("my-bucketname", "my-prefixname", true, doneCh) { + if object.Err != nil { + fmt.Println(object.Err) + return + } + fmt.Println(object) + } + return +} diff --git a/examples/s3/makebucket.go b/examples/s3/makebucket.go new file mode 100644 index 0000000..419c96c --- /dev/null +++ b/examples/s3/makebucket.go @@ -0,0 +1,47 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + err = s3Client.MakeBucket("my-bucketname", "us-east-1") + if err != nil { + log.Fatalln(err) + } + log.Println("Success") +} diff --git a/examples/s3/presignedgetobject.go b/examples/s3/presignedgetobject.go new file mode 100644 index 0000000..fd7fb9e --- /dev/null +++ b/examples/s3/presignedgetobject.go @@ -0,0 +1,54 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "net/url" + "time" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // Set request parameters + reqParams := make(url.Values) + reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") + + // Gernerate presigned get object url. + presignedURL, err := s3Client.PresignedGetObject("my-bucketname", "my-objectname", time.Duration(1000)*time.Second, reqParams) + if err != nil { + log.Fatalln(err) + } + log.Println(presignedURL) +} diff --git a/examples/s3/presignedheadobject.go b/examples/s3/presignedheadobject.go new file mode 100644 index 0000000..8dbc0a4 --- /dev/null +++ b/examples/s3/presignedheadobject.go @@ -0,0 +1,54 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "net/url" + "time" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // Set request parameters + reqParams := make(url.Values) + reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") + + // Gernerate presigned get object url. + presignedURL, err := s3Client.PresignedHeadObject("my-bucketname", "my-objectname", time.Duration(1000)*time.Second, reqParams) + if err != nil { + log.Fatalln(err) + } + log.Println(presignedURL) +} diff --git a/examples/s3/presignedpostpolicy.go b/examples/s3/presignedpostpolicy.go new file mode 100644 index 0000000..205ac95 --- /dev/null +++ b/examples/s3/presignedpostpolicy.go @@ -0,0 +1,60 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "log" + "time" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + policy := minio.NewPostPolicy() + policy.SetBucket("my-bucketname") + policy.SetKey("my-objectname") + // Expires in 10 days. + policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) + // Returns form data for POST form request. + url, formData, err := s3Client.PresignedPostPolicy(policy) + if err != nil { + log.Fatalln(err) + } + fmt.Printf("curl ") + for k, v := range formData { + fmt.Printf("-F %s=%s ", k, v) + } + fmt.Printf("-F file=@/etc/bash.bashrc ") + fmt.Printf("%s\n", url) +} diff --git a/examples/s3/presignedputobject.go b/examples/s3/presignedputobject.go new file mode 100644 index 0000000..b2f8b4f --- /dev/null +++ b/examples/s3/presignedputobject.go @@ -0,0 +1,48 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "time" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + presignedURL, err := s3Client.PresignedPutObject("my-bucketname", "my-objectname", time.Duration(1000)*time.Second) + if err != nil { + log.Fatalln(err) + } + log.Println(presignedURL) +} diff --git a/examples/s3/put-encrypted-object.go b/examples/s3/put-encrypted-object.go new file mode 100644 index 0000000..48b9367 --- /dev/null +++ b/examples/s3/put-encrypted-object.go @@ -0,0 +1,73 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "os" + + "github.com/minio/minio-go" + "github.com/minio/minio-go/pkg/encrypt" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + filePath := "my-testfile" // Specify a local file that we will upload + + // Open a local file that we will upload + file, err := os.Open(filePath) + if err != nil { + log.Fatalln(err) + } + defer file.Close() + + // Get file stats. + fstat, err := file.Stat() + if err != nil { + log.Fatalln(err) + } + + bucketname := "my-bucketname" // Specify a bucket name - the bucket must already exist + objectName := "my-objectname" // Specify a object name + password := "correct horse battery staple" // Specify your password. DO NOT USE THIS ONE - USE YOUR OWN. + + // New SSE-C where the cryptographic key is derived from a password and the objectname + bucketname as salt + encryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketname+objectName)) + + // Encrypt file content and upload to the server + n, err := s3Client.PutObject(bucketname, objectName, file, fstat.Size(), minio.PutObjectOptions{ServerSideEncryption: encryption}) + if err != nil { + log.Fatalln(err) + } + + log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") +} diff --git a/examples/s3/putobject-client-encryption.go b/examples/s3/putobject-client-encryption.go new file mode 100644 index 0000000..77d83b4 --- /dev/null +++ b/examples/s3/putobject-client-encryption.go @@ -0,0 +1,75 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "os" + "path" + + "github.com/minio/minio-go" + "github.com/minio/sio" + "golang.org/x/crypto/argon2" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + object, err := os.Open("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer object.Close() + objectStat, err := object.Stat() + if err != nil { + log.Fatalln(err) + } + + password := []byte("myfavoritepassword") // Change as per your needs. + salt := []byte(path.Join("my-bucketname", "my-objectname")) // Change as per your needs. + encrypted, err := sio.EncryptReader(object, sio.Config{ + // generate a 256 bit long key. + Key: argon2.IDKey(password, salt, 1, 64*1024, 4, 32), + }) + if err != nil { + log.Fatalln(err) + } + + encSize, err := sio.EncryptedSize(uint64(objectStat.Size())) + if err != nil { + log.Fatalln(err) + } + _, err = s3Client.PutObject("my-bucketname", "my-objectname", encrypted, int64(encSize), minio.PutObjectOptions{}) + if err != nil { + log.Fatalln(err) + } + log.Println("Successfully encrypted 'my-objectname'") +} diff --git a/examples/s3/putobject-context.go b/examples/s3/putobject-context.go new file mode 100644 index 0000000..acc923f --- /dev/null +++ b/examples/s3/putobject-context.go @@ -0,0 +1,68 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "os" + "time" + + "context" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + object, err := os.Open("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer object.Close() + + objectStat, err := object.Stat() + if err != nil { + log.Fatalln(err) + } + + n, err := s3Client.PutObjectWithContext(ctx, "my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ + ContentType: "application/octet-stream", + }) + if err != nil { + log.Fatalln(err) + } + log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") +} diff --git a/examples/s3/putobject-getobject-sse.go b/examples/s3/putobject-getobject-sse.go new file mode 100644 index 0000000..4e459b5 --- /dev/null +++ b/examples/s3/putobject-getobject-sse.go @@ -0,0 +1,68 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "bytes" + "io/ioutil" + "log" + + "github.com/minio/minio-go/pkg/encrypt" + + minio "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + minioClient, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + bucketName := "my-bucket" + objectName := "my-encrypted-object" + object := []byte("Hello again") + + encryption := encrypt.DefaultPBKDF([]byte("my secret password"), []byte(bucketName+objectName)) + _, err = minioClient.PutObject(bucketName, objectName, bytes.NewReader(object), int64(len(object)), minio.PutObjectOptions{ + ServerSideEncryption: encryption, + }) + if err != nil { + log.Fatalln(err) + } + + reader, err := minioClient.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: encryption}) + if err != nil { + log.Fatalln(err) + } + defer reader.Close() + + decBytes, err := ioutil.ReadAll(reader) + if err != nil { + log.Fatalln(err) + } + if !bytes.Equal(decBytes, object) { + log.Fatalln("Expected %s, got %s", string(object), string(decBytes)) + } +} diff --git a/examples/s3/putobject-progress.go b/examples/s3/putobject-progress.go new file mode 100644 index 0000000..0e92dd6 --- /dev/null +++ b/examples/s3/putobject-progress.go @@ -0,0 +1,64 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/cheggaaa/pb" + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + reader, err := s3Client.GetObject("my-bucketname", "my-objectname", minio.GetObjectOptions{}) + if err != nil { + log.Fatalln(err) + } + defer reader.Close() + + objectInfo, err := reader.Stat() + if err != nil { + log.Fatalln(err) + } + + // Progress reader is notified as PutObject makes progress with + // the Reads inside. + progress := pb.New64(objectInfo.Size) + progress.Start() + n, err := s3Client.PutObject("my-bucketname", "my-objectname-progress", reader, objectInfo.Size, minio.PutObjectOptions{ContentType: "application/octet-stream", Progress: progress}) + + if err != nil { + log.Fatalln(err) + } + log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") +} diff --git a/examples/s3/putobject-s3-accelerate.go b/examples/s3/putobject-s3-accelerate.go new file mode 100644 index 0000000..06345cd --- /dev/null +++ b/examples/s3/putobject-s3-accelerate.go @@ -0,0 +1,62 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "os" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // Enable S3 transfer accelerate endpoint. + s3Client.SetS3TransferAccelerate("s3-accelerate.amazonaws.com") + + object, err := os.Open("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer object.Close() + + objectStat, err := object.Stat() + if err != nil { + log.Fatalln(err) + } + + n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + if err != nil { + log.Fatalln(err) + } + log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") +} diff --git a/examples/s3/putobject-streaming.go b/examples/s3/putobject-streaming.go new file mode 100644 index 0000000..85b78dd --- /dev/null +++ b/examples/s3/putobject-streaming.go @@ -0,0 +1,55 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "os" + + minio "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + object, err := os.Open("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer object.Close() + + n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, -1, minio.PutObjectOptions{}) + if err != nil { + log.Fatalln(err) + } + + log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") +} diff --git a/examples/s3/putobject.go b/examples/s3/putobject.go new file mode 100644 index 0000000..b9e4ff1 --- /dev/null +++ b/examples/s3/putobject.go @@ -0,0 +1,58 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "os" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + object, err := os.Open("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer object.Close() + objectStat, err := object.Stat() + if err != nil { + log.Fatalln(err) + } + + n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + if err != nil { + log.Fatalln(err) + } + log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") +} diff --git a/examples/s3/removeallbucketnotification.go b/examples/s3/removeallbucketnotification.go new file mode 100644 index 0000000..1186afa --- /dev/null +++ b/examples/s3/removeallbucketnotification.go @@ -0,0 +1,50 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + err = s3Client.RemoveAllBucketNotification("my-bucketname") + if err != nil { + log.Fatalln(err) + } + + log.Println("Bucket notification are successfully removed.") +} diff --git a/examples/s3/removebucket.go b/examples/s3/removebucket.go new file mode 100644 index 0000000..7a7737e --- /dev/null +++ b/examples/s3/removebucket.go @@ -0,0 +1,49 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // This operation will only work if your bucket is empty. + err = s3Client.RemoveBucket("my-bucketname") + if err != nil { + log.Fatalln(err) + } + log.Println("Success") + +} diff --git a/examples/s3/removeincompleteupload.go b/examples/s3/removeincompleteupload.go new file mode 100644 index 0000000..31cc879 --- /dev/null +++ b/examples/s3/removeincompleteupload.go @@ -0,0 +1,47 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + err = s3Client.RemoveIncompleteUpload("my-bucketname", "my-objectname") + if err != nil { + log.Fatalln(err) + } + log.Println("Success") +} diff --git a/examples/s3/removeobject.go b/examples/s3/removeobject.go new file mode 100644 index 0000000..7e58485 --- /dev/null +++ b/examples/s3/removeobject.go @@ -0,0 +1,46 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + err = s3Client.RemoveObject("my-bucketname", "my-objectname") + if err != nil { + log.Fatalln(err) + } + log.Println("Success") +} diff --git a/examples/s3/removeobjects.go b/examples/s3/removeobjects.go new file mode 100644 index 0000000..a581134 --- /dev/null +++ b/examples/s3/removeobjects.go @@ -0,0 +1,71 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + objectsCh := make(chan string) + + // Send object names that are needed to be removed to objectsCh + go func() { + defer close(objectsCh) + + doneCh := make(chan struct{}) + + // Indicate to our routine to exit cleanly upon return. + defer close(doneCh) + + // List all objects from a bucket-name with a matching prefix. + for object := range s3Client.ListObjects("my-bucketname", "my-prefixname", true, doneCh) { + if object.Err != nil { + log.Fatalln(object.Err) + } + objectsCh <- object.Key + } + }() + + // Call RemoveObjects API + errorCh := s3Client.RemoveObjects("my-bucketname", objectsCh) + + // Print errors received from RemoveObjects API + for e := range errorCh { + log.Fatalln("Failed to remove " + e.ObjectName + ", error: " + e.Err.Error()) + } + + log.Println("Success") +} diff --git a/examples/s3/selectobject.go b/examples/s3/selectobject.go new file mode 100644 index 0000000..e23ccf8 --- /dev/null +++ b/examples/s3/selectobject.go @@ -0,0 +1,73 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "io" + "log" + "os" + + minio "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + + opts := minio.SelectObjectOptions{ + Expression: "select count(*) from s3object", + ExpressionType: minio.QueryExpressionTypeSQL, + InputSerialization: minio.SelectObjectInputSerialization{ + CompressionType: minio.SelectCompressionNONE, + CSV: &minio.CSVInputOptions{ + FileHeaderInfo: minio.CSVFileHeaderInfoNone, + RecordDelimiter: "\n", + FieldDelimiter: ",", + }, + }, + OutputSerialization: minio.SelectObjectOutputSerialization{ + CSV: &minio.CSVOutputOptions{ + RecordDelimiter: "\n", + FieldDelimiter: ",", + }, + }, + } + + reader, err := s3Client.SelectObjectContent(context.Background(), "mycsvbucket", "mycsv.csv", opts) + if err != nil { + log.Fatalln(err) + } + defer reader.Close() + + if _, err := io.Copy(os.Stdout, reader); err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/setbucketlifecycle.go b/examples/s3/setbucketlifecycle.go new file mode 100644 index 0000000..7eaa946 --- /dev/null +++ b/examples/s3/setbucketlifecycle.go @@ -0,0 +1,50 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Set lifecycle on a bucket + lifecycle := `expire-bucketEnabled365` + err = s3Client.SetBucketLifecycle("my-bucketname", lifecycle) + if err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/setbucketnotification.go b/examples/s3/setbucketnotification.go new file mode 100644 index 0000000..b5af30f --- /dev/null +++ b/examples/s3/setbucketnotification.go @@ -0,0 +1,86 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // ARN represents a notification channel that needs to be created in your S3 provider + // (e.g. http://docs.aws.amazon.com/sns/latest/dg/CreateTopic.html) + + // An example of an ARN: + // arn:aws:sns:us-east-1:804064459714:UploadPhoto + // ^ ^ ^ ^ ^ + // Provider __| | | | | + // | Region Account ID |_ Notification Name + // Service _| + // + // You should replace YOUR-PROVIDER, YOUR-SERVICE, YOUR-REGION, YOUR-ACCOUNT-ID and YOUR-RESOURCE + // with actual values that you receive from the S3 provider + + // Here you create a new Topic notification + topicArn := minio.NewArn("YOUR-PROVIDER", "YOUR-SERVICE", "YOUR-REGION", "YOUR-ACCOUNT-ID", "YOUR-RESOURCE") + topicConfig := minio.NewNotificationConfig(topicArn) + topicConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) + topicConfig.AddFilterPrefix("photos/") + topicConfig.AddFilterSuffix(".jpg") + + // Create a new Queue notification + queueArn := minio.NewArn("YOUR-PROVIDER", "YOUR-SERVICE", "YOUR-REGION", "YOUR-ACCOUNT-ID", "YOUR-RESOURCE") + queueConfig := minio.NewNotificationConfig(queueArn) + queueConfig.AddEvents(minio.ObjectRemovedAll) + + // Create a new Lambda (CloudFunction) + lambdaArn := minio.NewArn("YOUR-PROVIDER", "YOUR-SERVICE", "YOUR-REGION", "YOUR-ACCOUNT-ID", "YOUR-RESOURCE") + lambdaConfig := minio.NewNotificationConfig(lambdaArn) + lambdaConfig.AddEvents(minio.ObjectRemovedAll) + lambdaConfig.AddFilterSuffix(".swp") + + // Now, set all previously created notification configs + bucketNotification := minio.BucketNotification{} + bucketNotification.AddTopic(topicConfig) + bucketNotification.AddQueue(queueConfig) + bucketNotification.AddLambda(lambdaConfig) + + err = s3Client.SetBucketNotification("YOUR-BUCKET", bucketNotification) + if err != nil { + log.Fatalln("Error: " + err.Error()) + } + log.Println("Success") +} diff --git a/examples/s3/setbucketpolicy.go b/examples/s3/setbucketpolicy.go new file mode 100644 index 0000000..bc42da8 --- /dev/null +++ b/examples/s3/setbucketpolicy.go @@ -0,0 +1,51 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are + // dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // s3Client.TraceOn(os.Stderr) + + // Create policy + policy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::my-bucketname/*"],"Sid": ""}]}` + + err = s3Client.SetBucketPolicy("my-bucketname", policy) + if err != nil { + log.Fatalln(err) + } +} diff --git a/examples/s3/statobject.go b/examples/s3/statobject.go new file mode 100644 index 0000000..0b27a83 --- /dev/null +++ b/examples/s3/statobject.go @@ -0,0 +1,46 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname + // are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + stat, err := s3Client.StatObject("my-bucketname", "my-objectname", minio.StatObjectOptions{}) + if err != nil { + log.Fatalln(err) + } + log.Println(stat) +} diff --git a/functional_tests.go b/functional_tests.go new file mode 100644 index 0000000..62be0e9 --- /dev/null +++ b/functional_tests.go @@ -0,0 +1,7817 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "math/rand" + "mime/multipart" + "net/http" + "net/url" + "os" + "path/filepath" + "reflect" + "runtime" + "strconv" + "strings" + "time" + + humanize "github.com/dustin/go-humanize" + minio "github.com/minio/minio-go" + log "github.com/sirupsen/logrus" + + "github.com/minio/minio-go/pkg/encrypt" +) + +const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + return prefix + string(b[0:30-len(prefix)]) +} + +var dataFileMap = map[string]int{ + "datafile-1-b": 1, + "datafile-10-kB": 10 * humanize.KiByte, + "datafile-33-kB": 33 * humanize.KiByte, + "datafile-100-kB": 100 * humanize.KiByte, + "datafile-1.03-MB": 1056 * humanize.KiByte, + "datafile-1-MB": 1 * humanize.MiByte, + "datafile-5-MB": 5 * humanize.MiByte, + "datafile-6-MB": 6 * humanize.MiByte, + "datafile-11-MB": 11 * humanize.MiByte, + "datafile-65-MB": 65 * humanize.MiByte, +} + +func isFullMode() bool { + return os.Getenv("MINT_MODE") == "full" +} + +func getFuncName() string { + pc, _, _, _ := runtime.Caller(1) + return strings.TrimPrefix(runtime.FuncForPC(pc).Name(), "main.") +} + +// Tests bucket re-create errors. +func testMakeBucketError() { + region := "eu-central-1" + + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "MakeBucket(bucketName, region)" + // initialize logging params + args := map[string]interface{}{ + "bucketName": "", + "region": region, + } + + // skipping region functional tests for non s3 runs + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket in 'eu-central-1'. + if err = c.MakeBucket(bucketName, region); err != nil { + logError(testName, function, args, startTime, "", "MakeBucket Failed", err) + return + } + if err = c.MakeBucket(bucketName, region); err == nil { + logError(testName, function, args, startTime, "", "Bucket already exists", err) + return + } + // Verify valid error response from server. + if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && + minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { + logError(testName, function, args, startTime, "", "Invalid error returned by server", err) + return + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +func testMetadataSizeLimit() { + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader, objectSize, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "opts.UserMetadata": "", + } + rand.Seed(startTime.Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client creation failed", err) + return + } + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + const HeaderSizeLimit = 8 * 1024 + const UserMetadataLimit = 2 * 1024 + + // Meta-data greater than the 2 KB limit of AWS - PUT calls with this meta-data should fail + metadata := make(map[string]string) + metadata["X-Amz-Meta-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+UserMetadataLimit-len("X-Amz-Meta-Mint-Test"))) + args["metadata"] = fmt.Sprint(metadata) + + _, err = c.PutObject(bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) + if err == nil { + logError(testName, function, args, startTime, "", "Created object with user-defined metadata exceeding metadata size limits", nil) + return + } + + // Meta-data (headers) greater than the 8 KB limit of AWS - PUT calls with this meta-data should fail + metadata = make(map[string]string) + metadata["X-Amz-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+HeaderSizeLimit-len("X-Amz-Mint-Test"))) + args["metadata"] = fmt.Sprint(metadata) + _, err = c.PutObject(bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) + if err == nil { + logError(testName, function, args, startTime, "", "Created object with headers exceeding header size limits", nil) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Tests various bucket supported formats. +func testMakeBucketRegions() { + region := "eu-central-1" + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "MakeBucket(bucketName, region)" + // initialize logging params + args := map[string]interface{}{ + "bucketName": "", + "region": region, + } + + // skipping region functional tests for non s3 runs + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket in 'eu-central-1'. + if err = c.MakeBucket(bucketName, region); err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + // Make a new bucket with '.' in its name, in 'us-west-2'. This + // request is internally staged into a path style instead of + // virtual host style. + region = "us-west-2" + args["region"] = region + if err = c.MakeBucket(bucketName+".withperiod", region); err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName+".withperiod", c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +// Test PutObject using a large data to trigger multipart readat +func testPutObjectReadAt() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "opts": "objectContentType", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + bufSize := dataFileMap["datafile-65-MB"] + var reader = getDataReader("datafile-65-MB") + defer reader.Close() + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Object content type + objectContentType := "binary/octet-stream" + args["objectContentType"] = objectContentType + + n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: objectContentType}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "Get Object failed", err) + return + } + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat Object failed", err) + return + } + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Number of bytes in stat does not match, expected %d got %d", bufSize, st.Size), err) + return + } + if st.ContentType != objectContentType && st.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "Content types don't match", err) + return + } + if err := r.Close(); err != nil { + logError(testName, function, args, startTime, "", "Object Close failed", err) + return + } + if err := r.Close(); err == nil { + logError(testName, function, args, startTime, "", "Object is already closed, didn't return error on Close", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test PutObject using a large data to trigger multipart readat +func testPutObjectWithMetadata() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader,size, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", + } + + if !isFullMode() { + ignoredLog(testName, function, args, startTime, "Skipping functional tests for short/quick runs").Info() + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + bufSize := dataFileMap["datafile-65-MB"] + var reader = getDataReader("datafile-65-MB") + defer reader.Close() + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Object custom metadata + customContentType := "custom/contenttype" + + args["metadata"] = map[string][]string{ + "Content-Type": {customContentType}, + } + + n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ + ContentType: customContentType}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match, expected "+string(bufSize)+" got "+string(n), err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) + return + } + if st.ContentType != customContentType && st.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "ContentType does not match, expected "+customContentType+" got "+st.ContentType, err) + return + } + if err := r.Close(); err != nil { + logError(testName, function, args, startTime, "", "Object Close failed", err) + return + } + if err := r.Close(); err == nil { + logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testPutObjectWithContentLanguage() { + // initialize logging params + objectName := "test-object" + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader, size, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": objectName, + "size": -1, + "opts": "", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + data := bytes.Repeat([]byte("a"), int(0)) + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{ + ContentLanguage: "en", + }) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != 0 { + logError(testName, function, args, startTime, "", "Expected upload object '0' doesn't match with PutObject return value", err) + return + } + + objInfo, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + + if objInfo.Metadata.Get("Content-Language") != "en" { + logError(testName, function, args, startTime, "", "Expected content-language 'en' doesn't match with StatObject return value", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test put object with streaming signature. +func testPutObjectStreaming() { + // initialize logging params + objectName := "test-object" + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader,size,opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": objectName, + "size": -1, + "opts": "", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Upload an object. + sizes := []int64{0, 64*1024 - 1, 64 * 1024} + + for _, size := range sizes { + data := bytes.Repeat([]byte("a"), int(size)) + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(data), int64(size), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) + return + } + + if n != size { + logError(testName, function, args, startTime, "", "Expected upload object size doesn't match with PutObjectStreaming return value", err) + return + } + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test get object seeker from the end, using whence set to '2'. +func testGetObjectSeekEnd() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + buf, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(n), err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) + return + } + + pos, err := r.Seek(-100, 2) + if err != nil { + logError(testName, function, args, startTime, "", "Object Seek failed", err) + return + } + if pos != st.Size-100 { + logError(testName, function, args, startTime, "", "Incorrect position", err) + return + } + buf2 := make([]byte, 100) + m, err := io.ReadFull(r, buf2) + if err != nil { + logError(testName, function, args, startTime, "", "Error reading through io.ReadFull", err) + return + } + if m != len(buf2) { + logError(testName, function, args, startTime, "", "Number of bytes dont match, expected "+string(len(buf2))+" got "+string(m), err) + return + } + hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:]) + hexBuf2 := fmt.Sprintf("%02x", buf2[:m]) + if hexBuf1 != hexBuf2 { + logError(testName, function, args, startTime, "", "Values at same index dont match", err) + return + } + pos, err = r.Seek(-100, 2) + if err != nil { + logError(testName, function, args, startTime, "", "Object Seek failed", err) + return + } + if pos != st.Size-100 { + logError(testName, function, args, startTime, "", "Incorrect position", err) + return + } + if err = r.Close(); err != nil { + logError(testName, function, args, startTime, "", "ObjectClose failed", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test get object reader to not throw error on being closed twice. +func testGetObjectClosedTwice() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "PutObject response doesn't match sent bytes, expected "+string(int64(bufSize))+" got "+string(n), err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) + return + } + if err := r.Close(); err != nil { + logError(testName, function, args, startTime, "", "Object Close failed", err) + return + } + if err := r.Close(); err == nil { + logError(testName, function, args, startTime, "", "Already closed object. No error returned", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test RemoveObjectsWithContext request context cancels after timeout +func testRemoveObjectsWithContext() { + // Initialize logging params. + startTime := time.Now() + testName := getFuncName() + function := "RemoveObjectsWithContext(ctx, bucketName, objectsCh)" + args := map[string]interface{}{ + "bucketName": "", + } + + // Seed random based on current tie. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + } + + // Generate put data. + r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) + + // Multi remove of 20 objects. + nrObjects := 20 + objectsCh := make(chan string) + go func() { + defer close(objectsCh) + for i := 0; i < nrObjects; i++ { + objectName := "sample" + strconv.Itoa(i) + ".txt" + _, err = c.PutObject(bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + continue + } + objectsCh <- objectName + } + }() + // Set context to cancel in 1 nanosecond. + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + args["ctx"] = ctx + defer cancel() + + // Call RemoveObjectsWithContext API with short timeout. + errorCh := c.RemoveObjectsWithContext(ctx, bucketName, objectsCh) + // Check for error. + select { + case r := <-errorCh: + if r.Err == nil { + logError(testName, function, args, startTime, "", "RemoveObjectsWithContext should fail on short timeout", err) + return + } + } + // Set context with longer timeout. + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) + args["ctx"] = ctx + defer cancel() + // Perform RemoveObjectsWithContext with the longer timeout. Expect the removals to succeed. + errorCh = c.RemoveObjectsWithContext(ctx, bucketName, objectsCh) + select { + case r, more := <-errorCh: + if more || r.Err != nil { + logError(testName, function, args, startTime, "", "Unexpected error", r.Err) + return + } + } + + // Delete all objects and buckets. + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +// Test removing multiple objects with Remove API +func testRemoveMultipleObjects() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "RemoveObjects(bucketName, objectsCh)" + args := map[string]interface{}{ + "bucketName": "", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) + + // Multi remove of 1100 objects + nrObjects := 200 + + objectsCh := make(chan string) + + go func() { + defer close(objectsCh) + // Upload objects and send them to objectsCh + for i := 0; i < nrObjects; i++ { + objectName := "sample" + strconv.Itoa(i) + ".txt" + _, err = c.PutObject(bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + continue + } + objectsCh <- objectName + } + }() + + // Call RemoveObjects API + errorCh := c.RemoveObjects(bucketName, objectsCh) + + // Check if errorCh doesn't receive any error + select { + case r, more := <-errorCh: + if more { + logError(testName, function, args, startTime, "", "Unexpected error", r.Err) + return + } + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Tests FPutObject of a big file to trigger multipart +func testFPutObjectMultipart() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "FPutObject(bucketName, objectName, fileName, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "fileName": "", + "opts": "", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. + var fileName = getMintDataDirFilePath("datafile-65-MB") + if fileName == "" { + // Make a temp file with minPartSize bytes of data. + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + if err != nil { + logError(testName, function, args, startTime, "", "TempFile creation failed", err) + return + } + // Upload 2 parts to utilize all 3 'workers' in multipart and still have a part to upload. + if _, err = io.Copy(file, getDataReader("datafile-65-MB")); err != nil { + logError(testName, function, args, startTime, "", "Copy failed", err) + return + } + if err = file.Close(); err != nil { + logError(testName, function, args, startTime, "", "File Close failed", err) + return + } + fileName = file.Name() + args["fileName"] = fileName + } + totalSize := dataFileMap["datafile-65-MB"] + // Set base object name + objectName := bucketName + "FPutObject" + "-standard" + args["objectName"] = objectName + + objectContentType := "testapplication/octet-stream" + args["objectContentType"] = objectContentType + + // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) + n, err := c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType}) + if err != nil { + logError(testName, function, args, startTime, "", "FPutObject failed", err) + return + } + if n != int64(totalSize) { + logError(testName, function, args, startTime, "", "FPutObject failed", err) + return + } + + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + objInfo, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Unexpected error", err) + return + } + if objInfo.Size != int64(totalSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(totalSize))+" got "+string(objInfo.Size), err) + return + } + if objInfo.ContentType != objectContentType && objInfo.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "ContentType doesn't match", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Tests FPutObject with null contentType (default = application/octet-stream) +func testFPutObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "FPutObject(bucketName, objectName, fileName, opts)" + + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "fileName": "", + "opts": "", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + location := "us-east-1" + + // Make a new bucket. + args["bucketName"] = bucketName + args["location"] = location + function = "MakeBucket()bucketName, location" + err = c.MakeBucket(bucketName, location) + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. + // Use different data in part for multipart tests to check parts are uploaded in correct order. + var fName = getMintDataDirFilePath("datafile-65-MB") + if fName == "" { + // Make a temp file with minPartSize bytes of data. + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + if err != nil { + logError(testName, function, args, startTime, "", "TempFile creation failed", err) + return + } + + // Upload 3 parts to utilize all 3 'workers' in multipart and still have a part to upload. + if _, err = io.Copy(file, getDataReader("datafile-65-MB")); err != nil { + logError(testName, function, args, startTime, "", "File copy failed", err) + return + } + // Close the file pro-actively for windows. + if err = file.Close(); err != nil { + logError(testName, function, args, startTime, "", "File close failed", err) + return + } + defer os.Remove(file.Name()) + fName = file.Name() + } + totalSize := dataFileMap["datafile-65-MB"] + + // Set base object name + function = "FPutObject(bucketName, objectName, fileName, opts)" + objectName := bucketName + "FPutObject" + args["objectName"] = objectName + "-standard" + args["fileName"] = fName + args["opts"] = minio.PutObjectOptions{ContentType: "application/octet-stream"} + + // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) + n, err := c.FPutObject(bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + + if err != nil { + logError(testName, function, args, startTime, "", "FPutObject failed", err) + return + } + if n != int64(totalSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) + return + } + + // Perform FPutObject with no contentType provided (Expecting application/octet-stream) + args["objectName"] = objectName + "-Octet" + n, err = c.FPutObject(bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "File close failed", err) + return + } + if n != int64(totalSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) + return + } + srcFile, err := os.Open(fName) + if err != nil { + logError(testName, function, args, startTime, "", "File open failed", err) + return + } + defer srcFile.Close() + // Add extension to temp file name + tmpFile, err := os.Create(fName + ".gtar") + if err != nil { + logError(testName, function, args, startTime, "", "File create failed", err) + return + } + defer tmpFile.Close() + _, err = io.Copy(tmpFile, srcFile) + if err != nil { + logError(testName, function, args, startTime, "", "File copy failed", err) + return + } + + // Perform FPutObject with no contentType provided (Expecting application/x-gtar) + args["objectName"] = objectName + "-GTar" + args["opts"] = minio.PutObjectOptions{} + n, err = c.FPutObject(bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "FPutObject failed", err) + return + } + if n != int64(totalSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) + return + } + + // Check headers + function = "StatObject(bucketName, objectName, opts)" + args["objectName"] = objectName + "-standard" + rStandard, err := c.StatObject(bucketName, objectName+"-standard", minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + if rStandard.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rStandard.ContentType, err) + return + } + + function = "StatObject(bucketName, objectName, opts)" + args["objectName"] = objectName + "-Octet" + rOctet, err := c.StatObject(bucketName, objectName+"-Octet", minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + if rOctet.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rOctet.ContentType, err) + return + } + + function = "StatObject(bucketName, objectName, opts)" + args["objectName"] = objectName + "-GTar" + rGTar, err := c.StatObject(bucketName, objectName+"-GTar", minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "ContentType does not match, expected application/x-gtar or application/octet-stream, got "+rGTar.ContentType, err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + if err = os.Remove(fName + ".gtar"); err != nil { + logError(testName, function, args, startTime, "", "File remove failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Tests FPutObjectWithContext request context cancels after timeout +func testFPutObjectWithContext() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "FPutObject(bucketName, objectName, fileName, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "fileName": "", + "opts": "", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Upload 1 parts worth of data to use multipart upload. + // Use different data in part for multipart tests to check parts are uploaded in correct order. + var fName = getMintDataDirFilePath("datafile-1-MB") + if fName == "" { + // Make a temp file with 1 MiB bytes of data. + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectWithContextTest") + if err != nil { + logError(testName, function, args, startTime, "", "TempFile creation failed", err) + return + } + + // Upload 1 parts to trigger multipart upload + if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil { + logError(testName, function, args, startTime, "", "File copy failed", err) + return + } + // Close the file pro-actively for windows. + if err = file.Close(); err != nil { + logError(testName, function, args, startTime, "", "File close failed", err) + return + } + defer os.Remove(file.Name()) + fName = file.Name() + } + totalSize := dataFileMap["datafile-1-MB"] + + // Set base object name + objectName := bucketName + "FPutObjectWithContext" + args["objectName"] = objectName + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + args["ctx"] = ctx + defer cancel() + + // Perform standard FPutObjectWithContext with contentType provided (Expecting application/octet-stream) + _, err = c.FPutObjectWithContext(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + if err == nil { + logError(testName, function, args, startTime, "", "FPutObjectWithContext should fail on short timeout", err) + return + } + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) + defer cancel() + // Perform FPutObjectWithContext with a long timeout. Expect the put object to succeed + n, err := c.FPutObjectWithContext(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "FPutObjectWithContext shouldn't fail on long timeout", err) + return + } + if n != int64(totalSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(totalSize)+", got "+string(n), err) + return + } + + _, err = c.StatObject(bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() + +} + +// Tests FPutObjectWithContext request context cancels after timeout +func testFPutObjectWithContextV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "FPutObjectWithContext(ctx, bucketName, objectName, fileName, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "opts": "minio.PutObjectOptions{ContentType:objectContentType}", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Upload 1 parts worth of data to use multipart upload. + // Use different data in part for multipart tests to check parts are uploaded in correct order. + var fName = getMintDataDirFilePath("datafile-1-MB") + if fName == "" { + // Make a temp file with 1 MiB bytes of data. + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectWithContextTest") + if err != nil { + logError(testName, function, args, startTime, "", "Temp file creation failed", err) + return + } + + // Upload 1 parts to trigger multipart upload + if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil { + logError(testName, function, args, startTime, "", "File copy failed", err) + return + } + + // Close the file pro-actively for windows. + if err = file.Close(); err != nil { + logError(testName, function, args, startTime, "", "File close failed", err) + return + } + defer os.Remove(file.Name()) + fName = file.Name() + } + totalSize := dataFileMap["datafile-1-MB"] + + // Set base object name + objectName := bucketName + "FPutObjectWithContext" + args["objectName"] = objectName + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + args["ctx"] = ctx + defer cancel() + + // Perform standard FPutObjectWithContext with contentType provided (Expecting application/octet-stream) + _, err = c.FPutObjectWithContext(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) + if err == nil { + logError(testName, function, args, startTime, "", "FPutObjectWithContext should fail on short timeout", err) + return + } + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) + defer cancel() + // Perform FPutObjectWithContext with a long timeout. Expect the put object to succeed + n, err := c.FPutObjectWithContext(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "FPutObjectWithContext shouldn't fail on longer timeout", err) + return + } + if n != int64(totalSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match:wanted"+string(totalSize)+" got "+string(n), err) + return + } + + _, err = c.StatObject(bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() + +} + +// Test validates putObject with context to see if request cancellation is honored. +func testPutObjectWithContext() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObjectWithContext(ctx, bucketName, objectName, fileName, opts)" + args := map[string]interface{}{ + "ctx": "", + "bucketName": "", + "objectName": "", + "opts": "", + } + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket call failed", err) + return + } + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) + args["objectName"] = objectName + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + args["ctx"] = ctx + args["opts"] = minio.PutObjectOptions{ContentType: "binary/octet-stream"} + defer cancel() + + _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err == nil { + logError(testName, function, args, startTime, "", "PutObjectWithContext should fail on short timeout", err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) + args["ctx"] = ctx + + defer cancel() + reader = getDataReader("datafile-33-kB") + defer reader.Close() + _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectWithContext with long timeout failed", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() + +} + +// Tests get object ReaderSeeker interface methods. +func testGetObjectReadSeekFunctional() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + defer func() { + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + }() + + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + buf, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + // Save the data + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat object failed", err) + return + } + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + return + } + + // This following function helps us to compare data from the reader after seek + // with the data from the original buffer + cmpData := func(r io.Reader, start, end int) { + if end-start == 0 { + return + } + buffer := bytes.NewBuffer([]byte{}) + if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "CopyN failed", err) + return + } + } + if !bytes.Equal(buf[start:end], buffer.Bytes()) { + logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) + return + } + } + + // Generic seek error for errors other than io.EOF + seekErr := errors.New("seek error") + + testCases := []struct { + offset int64 + whence int + pos int64 + err error + shouldCmp bool + start int + end int + }{ + // Start from offset 0, fetch data and compare + {0, 0, 0, nil, true, 0, 0}, + // Start from offset 2048, fetch data and compare + {2048, 0, 2048, nil, true, 2048, bufSize}, + // Start from offset larger than possible + {int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0}, + // Move to offset 0 without comparing + {0, 0, 0, nil, false, 0, 0}, + // Move one step forward and compare + {1, 1, 1, nil, true, 1, bufSize}, + // Move larger than possible + {int64(bufSize), 1, 0, seekErr, false, 0, 0}, + // Provide negative offset with CUR_SEEK + {int64(-1), 1, 0, seekErr, false, 0, 0}, + // Test with whence SEEK_END and with positive offset + {1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0}, + // Test with whence SEEK_END and with negative offset + {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, + // Test with whence SEEK_END and with large negative offset + {-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0}, + } + + for i, testCase := range testCases { + // Perform seek operation + n, err := r.Seek(testCase.offset, testCase.whence) + // We expect an error + if testCase.err == seekErr && err == nil { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) + return + } + // We expect a specific error + if testCase.err != seekErr && testCase.err != err { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) + return + } + // If we expect an error go to the next loop + if testCase.err != nil { + continue + } + // Check the returned seek pos + if n != testCase.pos { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", number of bytes seeked does not match, expected "+string(testCase.pos)+", got "+string(n), err) + return + } + // Compare only if shouldCmp is activated + if testCase.shouldCmp { + cmpData(r, testCase.start, testCase.end) + } + } + successLogger(testName, function, args, startTime).Info() +} + +// Tests get object ReaderAt interface methods. +func testGetObjectReadAtFunctional() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + buf, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + // Save the data + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) + return + } + + // read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + offset := int64(2048) + + // read directly + buf1 := make([]byte, 512) + buf2 := make([]byte, 512) + buf3 := make([]byte, 512) + buf4 := make([]byte, 512) + + // Test readAt before stat is called such that objectInfo doesn't change. + m, err := r.ReadAt(buf1, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf1) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) + return + } + if !bytes.Equal(buf1, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + offset += 512 + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + return + } + + m, err = r.ReadAt(buf2, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf2) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) + return + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + + offset += 512 + m, err = r.ReadAt(buf3, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf3) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) + return + } + if !bytes.Equal(buf3, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + offset += 512 + m, err = r.ReadAt(buf4, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf4) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) + return + } + if !bytes.Equal(buf4, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + + buf5 := make([]byte, n) + // Read the whole object. + m, err = r.ReadAt(buf5, 0) + if err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + } + if m != len(buf5) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) + return + } + if !bytes.Equal(buf, buf5) { + logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) + return + } + + buf6 := make([]byte, n+1) + // Read the whole object and beyond. + _, err = r.ReadAt(buf6, 0) + if err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +// Test Presigned Post Policy +func testPresignedPostPolicy() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PresignedPostPolicy(policy)" + args := map[string]interface{}{ + "policy": "", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + // Azure requires the key to not start with a number + metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user") + metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") + + buf, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + // Save the data + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) + return + } + + policy := minio.NewPostPolicy() + + if err := policy.SetBucket(""); err == nil { + logError(testName, function, args, startTime, "", "SetBucket did not fail for invalid conditions", err) + return + } + if err := policy.SetKey(""); err == nil { + logError(testName, function, args, startTime, "", "SetKey did not fail for invalid conditions", err) + return + } + if err := policy.SetKeyStartsWith(""); err == nil { + logError(testName, function, args, startTime, "", "SetKeyStartsWith did not fail for invalid conditions", err) + return + } + if err := policy.SetExpires(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)); err == nil { + logError(testName, function, args, startTime, "", "SetExpires did not fail for invalid conditions", err) + return + } + if err := policy.SetContentType(""); err == nil { + logError(testName, function, args, startTime, "", "SetContentType did not fail for invalid conditions", err) + return + } + if err := policy.SetContentLengthRange(1024*1024, 1024); err == nil { + logError(testName, function, args, startTime, "", "SetContentLengthRange did not fail for invalid conditions", err) + return + } + if err := policy.SetUserMetadata("", ""); err == nil { + logError(testName, function, args, startTime, "", "SetUserMetadata did not fail for invalid conditions", err) + return + } + + policy.SetBucket(bucketName) + policy.SetKey(objectName) + policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days + policy.SetContentType("binary/octet-stream") + policy.SetContentLengthRange(10, 1024*1024) + policy.SetUserMetadata(metadataKey, metadataValue) + args["policy"] = policy.String() + + presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(policy) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err) + return + } + + var formBuf bytes.Buffer + writer := multipart.NewWriter(&formBuf) + for k, v := range formData { + writer.WriteField(k, v) + } + + // Get a 33KB file to upload and test if set post policy works + var filePath = getMintDataDirFilePath("datafile-33-kB") + if filePath == "" { + // Make a temp file with 33 KB data. + file, err := ioutil.TempFile(os.TempDir(), "PresignedPostPolicyTest") + if err != nil { + logError(testName, function, args, startTime, "", "TempFile creation failed", err) + return + } + if _, err = io.Copy(file, getDataReader("datafile-33-kB")); err != nil { + logError(testName, function, args, startTime, "", "Copy failed", err) + return + } + if err = file.Close(); err != nil { + logError(testName, function, args, startTime, "", "File Close failed", err) + return + } + filePath = file.Name() + } + + // add file to post request + f, err := os.Open(filePath) + defer f.Close() + if err != nil { + logError(testName, function, args, startTime, "", "File open failed", err) + return + } + w, err := writer.CreateFormFile("file", filePath) + if err != nil { + logError(testName, function, args, startTime, "", "CreateFormFile failed", err) + return + } + + _, err = io.Copy(w, f) + if err != nil { + logError(testName, function, args, startTime, "", "Copy failed", err) + return + } + writer.Close() + + // make post request with correct form data + res, err := http.Post(presignedPostPolicyURL.String(), writer.FormDataContentType(), bytes.NewReader(formBuf.Bytes())) + if err != nil { + logError(testName, function, args, startTime, "", "Http request failed", err) + return + } + defer res.Body.Close() + if res.StatusCode != http.StatusNoContent { + logError(testName, function, args, startTime, "", "Http request failed", errors.New(res.Status)) + return + } + + // expected path should be absolute path of the object + var scheme string + if mustParseBool(os.Getenv(enableHTTPS)) { + scheme = "https://" + } else { + scheme = "http://" + } + + expectedLocation := scheme + os.Getenv(serverEndpoint) + "/" + bucketName + "/" + objectName + expectedLocationBucketDNS := scheme + bucketName + "." + os.Getenv(serverEndpoint) + "/" + objectName + + if val, ok := res.Header["Location"]; ok { + if val[0] != expectedLocation && val[0] != expectedLocationBucketDNS { + logError(testName, function, args, startTime, "", "Location in header response is incorrect", err) + return + } + } else { + logError(testName, function, args, startTime, "", "Location not found in header response", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Tests copy object +func testCopyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(dst, src)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Make a new bucket in 'us-east-1' (destination bucket). + err = c.MakeBucket(bucketName+"-copy", "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) + return + } + + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + // Check the various fields of source object against destination object. + objInfo, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + + // Copy Source + src := minio.NewSourceInfo(bucketName, objectName, nil) + args["src"] = src + + // Set copy conditions. + + // All invalid conditions first. + err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) + if err == nil { + logError(testName, function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err) + return + } + err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) + if err == nil { + logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err) + return + } + err = src.SetMatchETagCond("") + if err == nil { + logError(testName, function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err) + return + } + err = src.SetMatchETagExceptCond("") + if err == nil { + logError(testName, function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err) + return + } + + err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + if err != nil { + logError(testName, function, args, startTime, "", "SetModifiedSinceCond failed", err) + return + } + err = src.SetMatchETagCond(objInfo.ETag) + if err != nil { + logError(testName, function, args, startTime, "", "SetMatchETagCond failed", err) + return + } + + dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) + args["dst"] = dst + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + + // Perform the Copy + err = c.CopyObject(dst, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) + return + } + + // Source object + r, err = c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + // Destination object + readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + // Check the various fields of source object against destination object. + objInfo, err = r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + objInfoCopy, err := readerCopy.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + if objInfo.Size != objInfoCopy.Size { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+", got "+string(objInfo.Size), err) + return + } + + // Close all the get readers before proceeding with CopyObject operations. + r.Close() + readerCopy.Close() + + // CopyObject again but with wrong conditions + src = minio.NewSourceInfo(bucketName, objectName, nil) + err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + if err != nil { + logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond failed", err) + return + } + err = src.SetMatchETagExceptCond(objInfo.ETag) + if err != nil { + logError(testName, function, args, startTime, "", "SetMatchETagExceptCond failed", err) + return + } + + // Perform the Copy which should fail + err = c.CopyObject(dst, src) + if err == nil { + logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) + return + } + + // Perform the Copy which should update only metadata. + src = minio.NewSourceInfo(bucketName, objectName, nil) + dst, err = minio.NewDestinationInfo(bucketName, objectName, nil, map[string]string{ + "Copy": "should be same", + }) + args["dst"] = dst + args["src"] = src + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + + err = c.CopyObject(dst, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject shouldn't fail", err) + return + } + + oi, err := c.StatObject(bucketName, objectName, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + + stOpts := minio.StatObjectOptions{} + stOpts.SetMatchETag(oi.ETag) + objInfo, err = c.StatObject(bucketName, objectName, stOpts) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject ETag should match and not fail", err) + return + } + + if objInfo.Metadata.Get("x-amz-meta-copy") != "should be same" { + logError(testName, function, args, startTime, "", "CopyObject modified metadata should match", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + if err = cleanupBucket(bucketName+"-copy", c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +// Tests SSE-C get object ReaderSeeker interface methods. +func testEncryptedGetObjectReadSeekFunctional() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + defer func() { + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + }() + + // Generate 65MiB of data. + bufSize := dataFileMap["datafile-65-MB"] + var reader = getDataReader("datafile-65-MB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + buf, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + // Save the data + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + ContentType: "binary/octet-stream", + ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), + }) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ + ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), + }) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + defer r.Close() + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat object failed", err) + return + } + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + return + } + + // This following function helps us to compare data from the reader after seek + // with the data from the original buffer + cmpData := func(r io.Reader, start, end int) { + if end-start == 0 { + return + } + buffer := bytes.NewBuffer([]byte{}) + if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "CopyN failed", err) + return + } + } + if !bytes.Equal(buf[start:end], buffer.Bytes()) { + logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) + return + } + } + + testCases := []struct { + offset int64 + whence int + pos int64 + err error + shouldCmp bool + start int + end int + }{ + // Start from offset 0, fetch data and compare + {0, 0, 0, nil, true, 0, 0}, + // Start from offset 2048, fetch data and compare + {2048, 0, 2048, nil, true, 2048, bufSize}, + // Start from offset larger than possible + {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, + // Move to offset 0 without comparing + {0, 0, 0, nil, false, 0, 0}, + // Move one step forward and compare + {1, 1, 1, nil, true, 1, bufSize}, + // Move larger than possible + {int64(bufSize), 1, 0, io.EOF, false, 0, 0}, + // Provide negative offset with CUR_SEEK + {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, + // Test with whence SEEK_END and with positive offset + {1024, 2, 0, io.EOF, false, 0, 0}, + // Test with whence SEEK_END and with negative offset + {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, + // Test with whence SEEK_END and with large negative offset + {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, + // Test with invalid whence + {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, + } + + for i, testCase := range testCases { + // Perform seek operation + n, err := r.Seek(testCase.offset, testCase.whence) + if err != nil && testCase.err == nil { + // We expected success. + logError(testName, function, args, startTime, "", + fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) + return + } + if err == nil && testCase.err != nil { + // We expected failure, but got success. + logError(testName, function, args, startTime, "", + fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) + return + } + if err != nil && testCase.err != nil { + if err.Error() != testCase.err.Error() { + // We expect a specific error + logError(testName, function, args, startTime, "", + fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) + return + } + } + // Check the returned seek pos + if n != testCase.pos { + logError(testName, function, args, startTime, "", + fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err) + return + } + // Compare only if shouldCmp is activated + if testCase.shouldCmp { + cmpData(r, testCase.start, testCase.end) + } + } + + successLogger(testName, function, args, startTime).Info() +} + +// Tests SSE-C get object ReaderAt interface methods. +func testEncryptedGetObjectReadAtFunctional() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate 65MiB of data. + bufSize := dataFileMap["datafile-65-MB"] + var reader = getDataReader("datafile-65-MB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + buf, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + // Save the data + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + ContentType: "binary/octet-stream", + ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), + }) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(n), err) + return + } + + // read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ + ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), + }) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + defer r.Close() + + offset := int64(2048) + + // read directly + buf1 := make([]byte, 512) + buf2 := make([]byte, 512) + buf3 := make([]byte, 512) + buf4 := make([]byte, 512) + + // Test readAt before stat is called such that objectInfo doesn't change. + m, err := r.ReadAt(buf1, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf1) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) + return + } + if !bytes.Equal(buf1, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + offset += 512 + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) + return + } + + m, err = r.ReadAt(buf2, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf2) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) + return + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + offset += 512 + m, err = r.ReadAt(buf3, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf3) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) + return + } + if !bytes.Equal(buf3, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + offset += 512 + m, err = r.ReadAt(buf4, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf4) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) + return + } + if !bytes.Equal(buf4, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + + buf5 := make([]byte, n) + // Read the whole object. + m, err = r.ReadAt(buf5, 0) + if err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + } + if m != len(buf5) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) + return + } + if !bytes.Equal(buf, buf5) { + logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) + return + } + + buf6 := make([]byte, n+1) + // Read the whole object and beyond. + _, err = r.ReadAt(buf6, 0) + if err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +// TestEncryptionPutGet tests client side encryption +func testEncryptionPutGet() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutEncryptedObject(bucketName, objectName, reader, sse)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "sse": "", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + testCases := []struct { + buf []byte + }{ + {buf: bytes.Repeat([]byte("F"), 1)}, + {buf: bytes.Repeat([]byte("F"), 15)}, + {buf: bytes.Repeat([]byte("F"), 16)}, + {buf: bytes.Repeat([]byte("F"), 17)}, + {buf: bytes.Repeat([]byte("F"), 31)}, + {buf: bytes.Repeat([]byte("F"), 32)}, + {buf: bytes.Repeat([]byte("F"), 33)}, + {buf: bytes.Repeat([]byte("F"), 1024)}, + {buf: bytes.Repeat([]byte("F"), 1024*2)}, + {buf: bytes.Repeat([]byte("F"), 1024*1024)}, + } + + const password = "correct horse battery staple" // https://xkcd.com/936/ + + for i, testCase := range testCases { + // Generate a random object name + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Secured object + sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) + args["sse"] = sse + + // Put encrypted data + _, err = c.PutObject(bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) + if err != nil { + logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) + if err != nil { + logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) + return + } + defer r.Close() + + // Compare the sent object with the received one + recvBuffer := bytes.NewBuffer([]byte{}) + if _, err = io.Copy(recvBuffer, r); err != nil { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) + return + } + if recvBuffer.Len() != len(testCase.buf) { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) + return + } + if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) + return + } + + successLogger(testName, function, args, startTime).Info() + + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// TestEncryptionFPut tests client side encryption +func testEncryptionFPut() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "filePath": "", + "contentType": "", + "sse": "", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Object custom metadata + customContentType := "custom/contenttype" + args["metadata"] = customContentType + + testCases := []struct { + buf []byte + }{ + {buf: bytes.Repeat([]byte("F"), 0)}, + {buf: bytes.Repeat([]byte("F"), 1)}, + {buf: bytes.Repeat([]byte("F"), 15)}, + {buf: bytes.Repeat([]byte("F"), 16)}, + {buf: bytes.Repeat([]byte("F"), 17)}, + {buf: bytes.Repeat([]byte("F"), 31)}, + {buf: bytes.Repeat([]byte("F"), 32)}, + {buf: bytes.Repeat([]byte("F"), 33)}, + {buf: bytes.Repeat([]byte("F"), 1024)}, + {buf: bytes.Repeat([]byte("F"), 1024*2)}, + {buf: bytes.Repeat([]byte("F"), 1024*1024)}, + } + + const password = "correct horse battery staple" // https://xkcd.com/936/ + for i, testCase := range testCases { + // Generate a random object name + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Secured object + sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) + args["sse"] = sse + + // Generate a random file name. + fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + file, err := os.Create(fileName) + if err != nil { + logError(testName, function, args, startTime, "", "file create failed", err) + return + } + _, err = file.Write(testCase.buf) + if err != nil { + logError(testName, function, args, startTime, "", "file write failed", err) + return + } + file.Close() + // Put encrypted data + if _, err = c.FPutObject(bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { + logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) + if err != nil { + logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) + return + } + defer r.Close() + + // Compare the sent object with the received one + recvBuffer := bytes.NewBuffer([]byte{}) + if _, err = io.Copy(recvBuffer, r); err != nil { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) + return + } + if recvBuffer.Len() != len(testCase.buf) { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) + return + } + if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { + logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) + return + } + + if err = os.Remove(fileName); err != nil { + logError(testName, function, args, startTime, "", "File remove failed", err) + return + } + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testBucketNotification() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "SetBucketNotification(bucketName)" + args := map[string]interface{}{ + "bucketName": "", + } + + if os.Getenv("NOTIFY_BUCKET") == "" || + os.Getenv("NOTIFY_SERVICE") == "" || + os.Getenv("NOTIFY_REGION") == "" || + os.Getenv("NOTIFY_ACCOUNTID") == "" || + os.Getenv("NOTIFY_RESOURCE") == "" { + ignoredLog(testName, function, args, startTime, "Skipped notification test as it is not configured").Info() + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + bucketName := os.Getenv("NOTIFY_BUCKET") + args["bucketName"] = bucketName + + topicArn := minio.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE")) + queueArn := minio.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource") + + topicConfig := minio.NewNotificationConfig(topicArn) + + topicConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) + topicConfig.AddFilterSuffix("jpg") + + queueConfig := minio.NewNotificationConfig(queueArn) + queueConfig.AddEvents(minio.ObjectCreatedAll) + queueConfig.AddFilterPrefix("photos/") + + bNotification := minio.BucketNotification{} + bNotification.AddTopic(topicConfig) + + // Add the same topicConfig again, should have no effect + // because it is duplicated + bNotification.AddTopic(topicConfig) + if len(bNotification.TopicConfigs) != 1 { + logError(testName, function, args, startTime, "", "Duplicate entry added", err) + return + } + + // Add and remove a queue config + bNotification.AddQueue(queueConfig) + bNotification.RemoveQueueByArn(queueArn) + + err = c.SetBucketNotification(bucketName, bNotification) + if err != nil { + logError(testName, function, args, startTime, "", "SetBucketNotification failed", err) + return + } + + bNotification, err = c.GetBucketNotification(bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "GetBucketNotification failed", err) + return + } + + if len(bNotification.TopicConfigs) != 1 { + logError(testName, function, args, startTime, "", "Topic config is empty", err) + return + } + + if bNotification.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" { + logError(testName, function, args, startTime, "", "Couldn't get the suffix", err) + return + } + + err = c.RemoveAllBucketNotification(bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "RemoveAllBucketNotification failed", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Tests comprehensive list of all methods. +func testFunctional() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "testFunctional()" + functionAll := "" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, nil, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + + // Make a new bucket. + function = "MakeBucket(bucketName, region)" + functionAll = "MakeBucket(bucketName, region)" + args["bucketName"] = bucketName + err = c.MakeBucket(bucketName, "us-east-1") + + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate a random file name. + fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + file, err := os.Create(fileName) + if err != nil { + logError(testName, function, args, startTime, "", "File creation failed", err) + return + } + for i := 0; i < 3; i++ { + buf := make([]byte, rand.Intn(1<<19)) + _, err = file.Write(buf) + if err != nil { + logError(testName, function, args, startTime, "", "File write failed", err) + return + } + } + file.Close() + + // Verify if bucket exits and you have access. + var exists bool + function = "BucketExists(bucketName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + } + exists, err = c.BucketExists(bucketName) + + if err != nil { + logError(testName, function, args, startTime, "", "BucketExists failed", err) + return + } + if !exists { + logError(testName, function, args, startTime, "", "Could not find the bucket", err) + return + } + + // Asserting the default bucket policy. + function = "GetBucketPolicy(bucketName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + } + nilPolicy, err := c.GetBucketPolicy(bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) + return + } + if nilPolicy != "" { + logError(testName, function, args, startTime, "", "policy should be set to nil", err) + return + } + + // Set the bucket policy to 'public readonly'. + function = "SetBucketPolicy(bucketName, readOnlyPolicy)" + functionAll += ", " + function + + readOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` + args = map[string]interface{}{ + "bucketName": bucketName, + "bucketPolicy": readOnlyPolicy, + } + + err = c.SetBucketPolicy(bucketName, readOnlyPolicy) + if err != nil { + logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) + return + } + // should return policy `readonly`. + function = "GetBucketPolicy(bucketName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + } + _, err = c.GetBucketPolicy(bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) + return + } + + // Make the bucket 'public writeonly'. + function = "SetBucketPolicy(bucketName, writeOnlyPolicy)" + functionAll += ", " + function + + writeOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` + args = map[string]interface{}{ + "bucketName": bucketName, + "bucketPolicy": writeOnlyPolicy, + } + err = c.SetBucketPolicy(bucketName, writeOnlyPolicy) + + if err != nil { + logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) + return + } + // should return policy `writeonly`. + function = "GetBucketPolicy(bucketName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + } + + _, err = c.GetBucketPolicy(bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) + return + } + + // Make the bucket 'public read/write'. + function = "SetBucketPolicy(bucketName, readWritePolicy)" + functionAll += ", " + function + + readWritePolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` + + args = map[string]interface{}{ + "bucketName": bucketName, + "bucketPolicy": readWritePolicy, + } + err = c.SetBucketPolicy(bucketName, readWritePolicy) + + if err != nil { + logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) + return + } + // should return policy `readwrite`. + function = "GetBucketPolicy(bucketName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + } + _, err = c.GetBucketPolicy(bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) + return + } + + // List all buckets. + function = "ListBuckets()" + functionAll += ", " + function + args = nil + buckets, err := c.ListBuckets() + + if len(buckets) == 0 { + logError(testName, function, args, startTime, "", "Found bucket list to be empty", err) + return + } + if err != nil { + logError(testName, function, args, startTime, "", "ListBuckets failed", err) + return + } + + // Verify if previously created bucket is listed in list buckets. + bucketFound := false + for _, bucket := range buckets { + if bucket.Name == bucketName { + bucketFound = true + } + } + + // If bucket not found error out. + if !bucketFound { + logError(testName, function, args, startTime, "", "Bucket: "+bucketName+" not found", err) + return + } + + objectName := bucketName + "unique" + + // Generate data + buf := bytes.Repeat([]byte("f"), 1<<19) + + function = "PutObject(bucketName, objectName, reader, contentType)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "contentType": "", + } + + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(len(buf)) { + logError(testName, function, args, startTime, "", "Length doesn't match, expected "+string(int64(len(buf)))+" got "+string(n), err) + return + } + + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName + "-nolength", + "contentType": "binary/octet-stream", + } + + n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(len(buf)) { + logError(testName, function, args, startTime, "", "Length doesn't match, expected "+string(int64(len(buf)))+" got "+string(n), err) + return + } + + // Instantiate a done channel to close all listing. + doneCh := make(chan struct{}) + defer close(doneCh) + + objFound := false + isRecursive := true // Recursive is true. + + function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "isRecursive": isRecursive, + } + + for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err) + return + } + + objFound = false + isRecursive = true // Recursive is true. + function = "ListObjectsV2(bucketName, objectName, isRecursive, doneCh)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "isRecursive": isRecursive, + } + + for obj := range c.ListObjectsV2(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err) + return + } + + incompObjNotFound := true + + function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "isRecursive": isRecursive, + } + + for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { + if objIncompl.Key != "" { + incompObjNotFound = false + break + } + } + if !incompObjNotFound { + logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err) + return + } + + function = "GetObject(bucketName, objectName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + } + newReader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + newReadBytes, err := ioutil.ReadAll(newReader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + if !bytes.Equal(newReadBytes, buf) { + logError(testName, function, args, startTime, "", "GetObject bytes mismatch", err) + return + } + newReader.Close() + + function = "FGetObject(bucketName, objectName, fileName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "fileName": fileName + "-f", + } + err = c.FGetObject(bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) + + if err != nil { + logError(testName, function, args, startTime, "", "FGetObject failed", err) + return + } + + function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": "", + "expires": 3600 * time.Second, + } + if _, err = c.PresignedHeadObject(bucketName, "", 3600*time.Second, nil); err == nil { + logError(testName, function, args, startTime, "", "PresignedHeadObject success", err) + return + } + + // Generate presigned HEAD object url. + function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "expires": 3600 * time.Second, + } + presignedHeadURL, err := c.PresignedHeadObject(bucketName, objectName, 3600*time.Second, nil) + + if err != nil { + logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) + return + } + // Verify if presigned url works. + resp, err := http.Head(presignedHeadURL.String()) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err) + return + } + if resp.StatusCode != http.StatusOK { + logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect, status "+string(resp.StatusCode), err) + return + } + if resp.Header.Get("ETag") == "" { + logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err) + return + } + resp.Body.Close() + + function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": "", + "expires": 3600 * time.Second, + } + _, err = c.PresignedGetObject(bucketName, "", 3600*time.Second, nil) + if err == nil { + logError(testName, function, args, startTime, "", "PresignedGetObject success", err) + return + } + + // Generate presigned GET object url. + function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "expires": 3600 * time.Second, + } + presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) + + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) + return + } + + // Verify if presigned url works. + resp, err = http.Get(presignedGetURL.String()) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) + return + } + if resp.StatusCode != http.StatusOK { + logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) + return + } + newPresignedBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) + return + } + resp.Body.Close() + if !bytes.Equal(newPresignedBytes, buf) { + logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) + return + } + + // Set request parameters. + reqParams := make(url.Values) + reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "expires": 3600 * time.Second, + "reqParams": reqParams, + } + presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) + + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) + return + } + // Verify if presigned url works. + resp, err = http.Get(presignedGetURL.String()) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) + return + } + if resp.StatusCode != http.StatusOK { + logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) + return + } + newPresignedBytes, err = ioutil.ReadAll(resp.Body) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) + return + } + if !bytes.Equal(newPresignedBytes, buf) { + logError(testName, function, args, startTime, "", "Bytes mismatch for presigned GET URL", err) + return + } + if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { + logError(testName, function, args, startTime, "", "wrong Content-Disposition received "+string(resp.Header.Get("Content-Disposition")), err) + return + } + + function = "PresignedPutObject(bucketName, objectName, expires)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": "", + "expires": 3600 * time.Second, + } + _, err = c.PresignedPutObject(bucketName, "", 3600*time.Second) + if err == nil { + logError(testName, function, args, startTime, "", "PresignedPutObject success", err) + return + } + + function = "PresignedPutObject(bucketName, objectName, expires)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName + "-presigned", + "expires": 3600 * time.Second, + } + presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) + + if err != nil { + logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) + return + } + + buf = bytes.Repeat([]byte("g"), 1<<19) + + req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) + if err != nil { + logError(testName, function, args, startTime, "", "Couldn't make HTTP request with PresignedPutObject URL", err) + return + } + httpClient := &http.Client{ + // Setting a sensible time out of 30secs to wait for response + // headers. Request is pro-actively cancelled after 30secs + // with no response. + Timeout: 30 * time.Second, + Transport: http.DefaultTransport, + } + resp, err = httpClient.Do(req) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) + return + } + + newReader, err = c.GetObject(bucketName, objectName+"-presigned", minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject after PresignedPutObject failed", err) + return + } + + newReadBytes, err = ioutil.ReadAll(newReader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll after GetObject failed", err) + return + } + + if !bytes.Equal(newReadBytes, buf) { + logError(testName, function, args, startTime, "", "Bytes mismatch", err) + return + } + + function = "RemoveObject(bucketName, objectName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + } + err = c.RemoveObject(bucketName, objectName) + + if err != nil { + logError(testName, function, args, startTime, "", "RemoveObject failed", err) + return + } + args["objectName"] = objectName + "-f" + err = c.RemoveObject(bucketName, objectName+"-f") + + if err != nil { + logError(testName, function, args, startTime, "", "RemoveObject failed", err) + return + } + + args["objectName"] = objectName + "-nolength" + err = c.RemoveObject(bucketName, objectName+"-nolength") + + if err != nil { + logError(testName, function, args, startTime, "", "RemoveObject failed", err) + return + } + + args["objectName"] = objectName + "-presigned" + err = c.RemoveObject(bucketName, objectName+"-presigned") + + if err != nil { + logError(testName, function, args, startTime, "", "RemoveObject failed", err) + return + } + + function = "RemoveBucket(bucketName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + } + err = c.RemoveBucket(bucketName) + + if err != nil { + logError(testName, function, args, startTime, "", "RemoveBucket failed", err) + return + } + err = c.RemoveBucket(bucketName) + if err == nil { + logError(testName, function, args, startTime, "", "RemoveBucket did not fail for invalid bucket name", err) + return + } + if err.Error() != "The specified bucket does not exist" { + logError(testName, function, args, startTime, "", "RemoveBucket failed", err) + return + } + + if err = os.Remove(fileName); err != nil { + logError(testName, function, args, startTime, "", "File Remove failed", err) + return + } + if err = os.Remove(fileName + "-f"); err != nil { + logError(testName, function, args, startTime, "", "File Remove failed", err) + return + } + successLogger(testName, functionAll, args, startTime).Info() +} + +// Test for validating GetObject Reader* methods functioning when the +// object is modified in the object store. +func testGetObjectModified() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + defer c.RemoveBucket(bucketName) + + // Upload an object. + objectName := "myobject" + args["objectName"] = objectName + content := "helloworld" + _, err = c.PutObject(bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"}) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) + return + } + + defer c.RemoveObject(bucketName, objectName) + + reader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to GetObject "+objectName+", from bucket "+bucketName, err) + return + } + defer reader.Close() + + // Read a few bytes of the object. + b := make([]byte, 5) + n, err := reader.ReadAt(b, 0) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to read object "+objectName+", from bucket "+bucketName+" at an offset", err) + return + } + + // Upload different contents to the same object while object is being read. + newContent := "goodbyeworld" + _, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"}) + if err != nil { + logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) + return + } + + // Confirm that a Stat() call in between doesn't change the Object's cached etag. + _, err = reader.Stat() + expectedError := "At least one of the pre-conditions you specified did not hold" + if err.Error() != expectedError { + logError(testName, function, args, startTime, "", "Expected Stat to fail with error "+expectedError+", but received "+err.Error(), err) + return + } + + // Read again only to find object contents have been modified since last read. + _, err = reader.ReadAt(b, int64(n)) + if err.Error() != expectedError { + logError(testName, function, args, startTime, "", "Expected ReadAt to fail with error "+expectedError+", but received "+err.Error(), err) + return + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test validates putObject to upload a file seeked at a given offset. +func testPutObjectUploadSeekedObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, fileToUpload, contentType)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "fileToUpload": "", + "contentType": "binary/octet-stream", + } + + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + defer c.RemoveBucket(bucketName) + + var tempfile *os.File + + if fileName := getMintDataDirFilePath("datafile-100-kB"); fileName != "" { + tempfile, err = os.Open(fileName) + if err != nil { + logError(testName, function, args, startTime, "", "File open failed", err) + return + } + args["fileToUpload"] = fileName + } else { + tempfile, err = ioutil.TempFile("", "minio-go-upload-test-") + if err != nil { + logError(testName, function, args, startTime, "", "TempFile create failed", err) + return + } + args["fileToUpload"] = tempfile.Name() + + // Generate 100kB data + if _, err = io.Copy(tempfile, getDataReader("datafile-100-kB")); err != nil { + logError(testName, function, args, startTime, "", "File copy failed", err) + return + } + + defer os.Remove(tempfile.Name()) + + // Seek back to the beginning of the file. + tempfile.Seek(0, 0) + } + var length = 100 * humanize.KiByte + objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) + args["objectName"] = objectName + + offset := length / 2 + if _, err = tempfile.Seek(int64(offset), 0); err != nil { + logError(testName, function, args, startTime, "", "TempFile seek failed", err) + return + } + + n, err := c.PutObject(bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + if n != int64(length-offset) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid length returned, expected %d got %d", int64(length-offset), n), err) + return + } + tempfile.Close() + + obj, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + defer obj.Close() + + n, err = obj.Seek(int64(offset), 0) + if err != nil { + logError(testName, function, args, startTime, "", "Seek failed", err) + return + } + if n != int64(offset) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(offset), n), err) + return + } + + n, err = c.PutObject(bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + if n != int64(length-offset) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(length-offset), n), err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Tests bucket re-create errors. +func testMakeBucketErrorV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "MakeBucket(bucketName, region)" + args := map[string]interface{}{ + "bucketName": "", + "region": "eu-west-1", + } + + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + region := "eu-west-1" + args["bucketName"] = bucketName + args["region"] = region + + // Make a new bucket in 'eu-west-1'. + if err = c.MakeBucket(bucketName, region); err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + if err = c.MakeBucket(bucketName, region); err == nil { + logError(testName, function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err) + return + } + // Verify valid error response from server. + if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && + minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { + logError(testName, function, args, startTime, "", "Invalid error returned by server", err) + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test get object reader to not throw error on being closed twice. +func testGetObjectClosedTwiceV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "MakeBucket(bucketName, region)" + args := map[string]interface{}{ + "bucketName": "", + "region": "eu-west-1", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) + return + } + if err := r.Close(); err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + if err := r.Close(); err == nil { + logError(testName, function, args, startTime, "", "Object is already closed, should return error", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Tests FPutObject hidden contentType setting +func testFPutObjectV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "FPutObject(bucketName, objectName, fileName, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "fileName": "", + "opts": "", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Make a temp file with 11*1024*1024 bytes of data. + file, err := ioutil.TempFile(os.TempDir(), "FPutObjectTest") + if err != nil { + logError(testName, function, args, startTime, "", "TempFile creation failed", err) + return + } + + r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024)) + n, err := io.CopyN(file, r, 11*1024*1024) + if err != nil { + logError(testName, function, args, startTime, "", "Copy failed", err) + return + } + if n != int64(11*1024*1024) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) + return + } + + // Close the file pro-actively for windows. + err = file.Close() + if err != nil { + logError(testName, function, args, startTime, "", "File close failed", err) + return + } + + // Set base object name + objectName := bucketName + "FPutObject" + args["objectName"] = objectName + args["fileName"] = file.Name() + + // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) + n, err = c.FPutObject(bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "FPutObject failed", err) + return + } + if n != int64(11*1024*1024) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) + return + } + + // Perform FPutObject with no contentType provided (Expecting application/octet-stream) + args["objectName"] = objectName + "-Octet" + args["contentType"] = "" + + n, err = c.FPutObject(bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "FPutObject failed", err) + return + } + if n != int64(11*1024*1024) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) + return + } + + // Add extension to temp file name + fileName := file.Name() + err = os.Rename(file.Name(), fileName+".gtar") + if err != nil { + logError(testName, function, args, startTime, "", "Rename failed", err) + return + } + + // Perform FPutObject with no contentType provided (Expecting application/x-gtar) + args["objectName"] = objectName + "-Octet" + args["contentType"] = "" + args["fileName"] = fileName + ".gtar" + + n, err = c.FPutObject(bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "FPutObject failed", err) + return + } + if n != int64(11*1024*1024) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) + return + } + + // Check headers + rStandard, err := c.StatObject(bucketName, objectName+"-standard", minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + if rStandard.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rStandard.ContentType, err) + return + } + + rOctet, err := c.StatObject(bucketName, objectName+"-Octet", minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + if rOctet.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rOctet.ContentType, err) + return + } + + rGTar, err := c.StatObject(bucketName, objectName+"-GTar", minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" { + logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-gtar , got "+rGTar.ContentType, err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + err = os.Remove(fileName + ".gtar") + if err != nil { + logError(testName, function, args, startTime, "", "File remove failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +// Tests various bucket supported formats. +func testMakeBucketRegionsV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "MakeBucket(bucketName, region)" + args := map[string]interface{}{ + "bucketName": "", + "region": "eu-west-1", + } + + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket in 'eu-central-1'. + if err = c.MakeBucket(bucketName, "eu-west-1"); err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + // Make a new bucket with '.' in its name, in 'us-west-2'. This + // request is internally staged into a path style instead of + // virtual host style. + if err = c.MakeBucket(bucketName+".withperiod", "us-west-2"); err != nil { + args["bucketName"] = bucketName + ".withperiod" + args["region"] = "us-west-2" + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName+".withperiod", c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Tests get object ReaderSeeker interface methods. +func testGetObjectReadSeekFunctionalV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + buf, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + // Save the data. + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + defer r.Close() + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) + return + } + + offset := int64(2048) + n, err = r.Seek(offset, 0) + if err != nil { + logError(testName, function, args, startTime, "", "Seek failed", err) + return + } + if n != offset { + logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) + return + } + n, err = r.Seek(0, 1) + if err != nil { + logError(testName, function, args, startTime, "", "Seek failed", err) + return + } + if n != offset { + logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) + return + } + _, err = r.Seek(offset, 2) + if err == nil { + logError(testName, function, args, startTime, "", "Seek on positive offset for whence '2' should error out", err) + return + } + n, err = r.Seek(-offset, 2) + if err != nil { + logError(testName, function, args, startTime, "", "Seek failed", err) + return + } + if n != st.Size-offset { + logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(st.Size-offset)+" got "+string(n), err) + return + } + + var buffer1 bytes.Buffer + if _, err = io.CopyN(&buffer1, r, st.Size); err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "Copy failed", err) + return + } + } + if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) { + logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) + return + } + + // Seek again and read again. + n, err = r.Seek(offset-1, 0) + if err != nil { + logError(testName, function, args, startTime, "", "Seek failed", err) + return + } + if n != (offset - 1) { + logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset-1)+" got "+string(n), err) + return + } + + var buffer2 bytes.Buffer + if _, err = io.CopyN(&buffer2, r, st.Size); err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "Copy failed", err) + return + } + } + // Verify now lesser bytes. + if !bytes.Equal(buf[2047:], buffer2.Bytes()) { + logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Tests get object ReaderAt interface methods. +func testGetObjectReadAtFunctionalV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObject(bucketName, objectName)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + buf, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + + // Save the data + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(n), err) + return + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + defer r.Close() + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) + return + } + + offset := int64(2048) + + // Read directly + buf2 := make([]byte, 512) + buf3 := make([]byte, 512) + buf4 := make([]byte, 512) + + m, err := r.ReadAt(buf2, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf2) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+" got "+string(m), err) + return + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + offset += 512 + m, err = r.ReadAt(buf3, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf3) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+" got "+string(m), err) + return + } + if !bytes.Equal(buf3, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + offset += 512 + m, err = r.ReadAt(buf4, offset) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + if m != len(buf4) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+" got "+string(m), err) + return + } + if !bytes.Equal(buf4, buf[offset:offset+512]) { + logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) + return + } + + buf5 := make([]byte, n) + // Read the whole object. + m, err = r.ReadAt(buf5, 0) + if err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + } + if m != len(buf5) { + logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+" got "+string(m), err) + return + } + if !bytes.Equal(buf, buf5) { + logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) + return + } + + buf6 := make([]byte, n+1) + // Read the whole object and beyond. + _, err = r.ReadAt(buf6, 0) + if err != nil { + if err != io.EOF { + logError(testName, function, args, startTime, "", "ReadAt failed", err) + return + } + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Tests copy object +func testCopyObjectV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Make a new bucket in 'us-east-1' (destination bucket). + err = c.MakeBucket(bucketName+"-copy", "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate 33K of data. + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+" got "+string(n), err) + return + } + + r, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + // Check the various fields of source object against destination object. + objInfo, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + r.Close() + + // Copy Source + src := minio.NewSourceInfo(bucketName, objectName, nil) + args["source"] = src + + // Set copy conditions. + + // All invalid conditions first. + err = src.SetModifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) + if err == nil { + logError(testName, function, args, startTime, "", "SetModifiedSinceCond did not fail for invalid conditions", err) + return + } + err = src.SetUnmodifiedSinceCond(time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)) + if err == nil { + logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond did not fail for invalid conditions", err) + return + } + err = src.SetMatchETagCond("") + if err == nil { + logError(testName, function, args, startTime, "", "SetMatchETagCond did not fail for invalid conditions", err) + return + } + err = src.SetMatchETagExceptCond("") + if err == nil { + logError(testName, function, args, startTime, "", "SetMatchETagExceptCond did not fail for invalid conditions", err) + return + } + + err = src.SetModifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + if err != nil { + logError(testName, function, args, startTime, "", "SetModifiedSinceCond failed", err) + return + } + err = src.SetMatchETagCond(objInfo.ETag) + if err != nil { + logError(testName, function, args, startTime, "", "SetMatchETagCond failed", err) + return + } + + dst, err := minio.NewDestinationInfo(bucketName+"-copy", objectName+"-copy", nil, nil) + args["destination"] = dst + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + + // Perform the Copy + err = c.CopyObject(dst, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) + return + } + + // Source object + r, err = c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + // Destination object + readerCopy, err := c.GetObject(bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + // Check the various fields of source object against destination object. + objInfo, err = r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + objInfoCopy, err := readerCopy.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + if objInfo.Size != objInfoCopy.Size { + logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+" got "+string(objInfo.Size), err) + return + } + + // Close all the readers. + r.Close() + readerCopy.Close() + + // CopyObject again but with wrong conditions + src = minio.NewSourceInfo(bucketName, objectName, nil) + err = src.SetUnmodifiedSinceCond(time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC)) + if err != nil { + logError(testName, function, args, startTime, "", "SetUnmodifiedSinceCond failed", err) + return + } + err = src.SetMatchETagExceptCond(objInfo.ETag) + if err != nil { + logError(testName, function, args, startTime, "", "SetMatchETagExceptCond failed", err) + return + } + + // Perform the Copy which should fail + err = c.CopyObject(dst, src) + if err == nil { + logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + if err = cleanupBucket(bucketName+"-copy", c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +func testComposeObjectErrorCasesWrapper(c *minio.Client) { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "ComposeObject(destination, sourceList)" + args := map[string]interface{}{} + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") + + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Test that more than 10K source objects cannot be + // concatenated. + srcArr := [10001]minio.SourceInfo{} + srcSlice := srcArr[:] + dst, err := minio.NewDestinationInfo(bucketName, "object", nil, nil) + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + + args["destination"] = dst + // Just explain about srcArr in args["sourceList"] + // to stop having 10,001 null headers logged + args["sourceList"] = "source array of 10,001 elements" + if err := c.ComposeObject(dst, srcSlice); err == nil { + logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err) + return + } else if err.Error() != "There must be as least one and up to 10000 source objects." { + logError(testName, function, args, startTime, "", "Got unexpected error", err) + return + } + + // Create a source with invalid offset spec and check that + // error is returned: + // 1. Create the source object. + const badSrcSize = 5 * 1024 * 1024 + buf := bytes.Repeat([]byte("1"), badSrcSize) + _, err = c.PutObject(bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + // 2. Set invalid range spec on the object (going beyond + // object size) + badSrc := minio.NewSourceInfo(bucketName, "badObject", nil) + err = badSrc.SetRange(1, badSrcSize) + if err != nil { + logError(testName, function, args, startTime, "", "Setting NewSourceInfo failed", err) + return + } + // 3. ComposeObject call should fail. + if err := c.ComposeObject(dst, []minio.SourceInfo{badSrc}); err == nil { + logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err) + return + } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { + logError(testName, function, args, startTime, "", "Got invalid error", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test expected error cases +func testComposeObjectErrorCasesV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "ComposeObject(destination, sourceList)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + testComposeObjectErrorCasesWrapper(c) +} + +func testComposeMultipleSources(c *minio.Client) { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "ComposeObject(destination, sourceList)" + args := map[string]interface{}{ + "destination": "", + "sourceList": "", + } + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Upload a small source object + const srcSize = 1024 * 1024 * 5 + buf := bytes.Repeat([]byte("1"), srcSize) + _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + // We will append 10 copies of the object. + srcs := []minio.SourceInfo{} + for i := 0; i < 10; i++ { + srcs = append(srcs, minio.NewSourceInfo(bucketName, "srcObject", nil)) + } + // make the last part very small + err = srcs[9].SetRange(0, 0) + if err != nil { + logError(testName, function, args, startTime, "", "SetRange failed", err) + return + } + args["sourceList"] = srcs + + dst, err := minio.NewDestinationInfo(bucketName, "dstObject", nil, nil) + args["destination"] = dst + + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + err = c.ComposeObject(dst, srcs) + if err != nil { + logError(testName, function, args, startTime, "", "ComposeObject failed", err) + return + } + + objProps, err := c.StatObject(bucketName, "dstObject", minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject failed", err) + return + } + + if objProps.Size != 9*srcSize+1 { + logError(testName, function, args, startTime, "", "Size mismatched! Expected "+string(10000*srcSize)+" got "+string(objProps.Size), err) + return + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +// Test concatenating multiple objects objects +func testCompose10KSourcesV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "ComposeObject(destination, sourceList)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + testComposeMultipleSources(c) +} + +func testEncryptedEmptyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader, objectSize, opts)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + return + } + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + sse := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"object")) + + // 1. create an sse-c encrypted object to copy by uploading + const srcSize = 0 + var buf []byte // Empty buffer + args["objectName"] = "object" + _, err = c.PutObject(bucketName, "object", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + return + } + + // 2. Test CopyObject for an empty object + dstInfo, err := minio.NewDestinationInfo(bucketName, "new-object", sse, nil) + if err != nil { + args["objectName"] = "new-object" + function = "NewDestinationInfo(bucketName, objectName, sse, userMetadata)" + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + srcInfo := minio.NewSourceInfo(bucketName, "object", sse) + if err = c.CopyObject(dstInfo, srcInfo); err != nil { + function = "CopyObject(dstInfo, srcInfo)" + logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject failed", err) + return + } + + // 3. Test Key rotation + newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"new-object")) + dstInfo, err = minio.NewDestinationInfo(bucketName, "new-object", newSSE, nil) + if err != nil { + args["objectName"] = "new-object" + function = "NewDestinationInfo(bucketName, objectName, encryptSSEC, userMetadata)" + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + + srcInfo = minio.NewSourceInfo(bucketName, "new-object", sse) + if err = c.CopyObject(dstInfo, srcInfo); err != nil { + function = "CopyObject(dstInfo, srcInfo)" + logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject with key rotation failed", err) + return + } + + // 4. Download the object. + reader, err := c.GetObject(bucketName, "new-object", minio.GetObjectOptions{ServerSideEncryption: newSSE}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + defer reader.Close() + + decBytes, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, map[string]interface{}{}, startTime, "", "ReadAll failed", err) + return + } + if !bytes.Equal(decBytes, buf) { + logError(testName, function, map[string]interface{}{}, startTime, "", "Downloaded object doesn't match the empty encrypted object", err) + return + } + // Delete all objects and buckets + delete(args, "objectName") + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testEncryptedCopyObjectWrapper(c *minio.Client) { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) + sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) + + // 1. create an sse-c encrypted object to copy by uploading + const srcSize = 1024 * 1024 + buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB + _, err = c.PutObject(bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ + ServerSideEncryption: sseSrc, + }) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + return + } + + // 2. copy object and change encryption key + src := minio.NewSourceInfo(bucketName, "srcObject", sseSrc) + args["source"] = src + dst, err := minio.NewDestinationInfo(bucketName, "dstObject", sseDst, nil) + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + args["destination"] = dst + + err = c.CopyObject(dst, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) + return + } + + // 3. get copied object and check if content is equal + coreClient := minio.Core{c} + reader, _, err := coreClient.GetObject(bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: sseDst}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + decBytes, err := ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + if !bytes.Equal(decBytes, buf) { + logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) + return + } + reader.Close() + + // Test key rotation for source object in-place. + newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"srcObject")) // replace key + dst, err = minio.NewDestinationInfo(bucketName, "srcObject", newSSE, nil) + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + args["destination"] = dst + + err = c.CopyObject(dst, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) + return + } + + // Get copied object and check if content is equal + reader, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + decBytes, err = ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + if !bytes.Equal(decBytes, buf) { + logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) + return + } + reader.Close() + + // Test in-place decryption. + dst, err = minio.NewDestinationInfo(bucketName, "srcObject", nil, nil) + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + args["destination"] = dst + + src = minio.NewSourceInfo(bucketName, "srcObject", newSSE) + args["source"] = src + err = c.CopyObject(dst, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) + return + } + + // Get copied decrypted object and check if content is equal + reader, _, err = coreClient.GetObject(bucketName, "srcObject", minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + defer reader.Close() + + decBytes, err = ioutil.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + if !bytes.Equal(decBytes, buf) { + logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test encrypted copy object +func testEncryptedCopyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + // c.TraceOn(os.Stderr) + testEncryptedCopyObjectWrapper(c) +} + +// Test encrypted copy object +func testEncryptedCopyObjectV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + // c.TraceOn(os.Stderr) + testEncryptedCopyObjectWrapper(c) +} + +func testDecryptedCopyObject() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v2 client object creation failed", err) + return + } + + bucketName, objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-"), "object" + if err = c.MakeBucket(bucketName, "us-east-1"); err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + encryption := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)) + _, err = c.PutObject(bucketName, objectName, bytes.NewReader(bytes.Repeat([]byte("a"), 1024*1024)), 1024*1024, minio.PutObjectOptions{ + ServerSideEncryption: encryption, + }) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + return + } + + src := minio.NewSourceInfo(bucketName, objectName, encrypt.SSECopy(encryption)) + args["source"] = src + dst, err := minio.NewDestinationInfo(bucketName, "decrypted-"+objectName, nil, nil) + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + args["destination"] = dst + + if err = c.CopyObject(dst, src); err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) + return + } + if _, err = c.GetObject(bucketName, "decrypted-"+objectName, minio.GetObjectOptions{}); err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +// Test Core CopyObjectPart implementation +func testCoreEncryptedCopyObjectPart() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObjectPart(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + client, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + return + } + + // Instantiate new core client object. + c := minio.Core{client} + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + } + defer cleanupBucket(bucketName, client) + // Make a buffer with 5MB of data + buf := bytes.Repeat([]byte("abcde"), 1024*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + password := "correct horse battery staple" + srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) + + objInfo, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", map[string]string{ + "Content-Type": "binary/octet-stream", + }, srcencryption) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + } + + if objInfo.Size != int64(len(buf)) { + logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size), err) + } + + destBucketName := bucketName + destObjectName := objectName + "-dest" + dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) + + uploadID, err := c.NewMultipartUpload(destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) + if err != nil { + logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) + } + + // Content of the destination object will be two copies of + // `objectName` concatenated, followed by first byte of + // `objectName`. + metadata := make(map[string]string) + header := make(http.Header) + encrypt.SSECopy(srcencryption).Marshal(header) + dstencryption.Marshal(header) + for k, v := range header { + metadata[k] = v[0] + } + // First of three parts + fstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Second of three parts + sndPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Last of three parts + lstPart, err := c.CopyObjectPart(bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) + } + + // Complete the multipart upload + _, err = c.CompleteMultipartUpload(destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}) + if err != nil { + logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) + } + + // Stat the object and check its length matches + objInfo, err = c.StatObject(destBucketName, destObjectName, minio.StatObjectOptions{minio.GetObjectOptions{ServerSideEncryption: dstencryption}}) + if err != nil { + logError(testName, function, args, startTime, "", "StatObject call failed", err) + } + + if objInfo.Size != (5*1024*1024)*2+1 { + logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) + } + + // Now we read the data back + getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} + getOpts.SetRange(0, 5*1024*1024-1) + r, _, err := c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf := make([]byte, 5*1024*1024) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf, buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) + } + + getOpts.SetRange(5*1024*1024, 0) + r, _, err = c.GetObject(destBucketName, destObjectName, getOpts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject call failed", err) + } + getBuf = make([]byte, 5*1024*1024+1) + _, err = io.ReadFull(r, getBuf) + if err != nil { + logError(testName, function, args, startTime, "", "Read buffer failed", err) + } + if !bytes.Equal(getBuf[:5*1024*1024], buf) { + logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) + } + if getBuf[5*1024*1024] != buf[0] { + logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) + } + + successLogger(testName, function, args, startTime).Info() + + // Do not need to remove destBucketName its same as bucketName. +} +func testUserMetadataCopying() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + // c.TraceOn(os.Stderr) + testUserMetadataCopyingWrapper(c) +} + +func testUserMetadataCopyingWrapper(c *minio.Client) { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + // Make a new bucket in 'us-east-1' (source bucket). + err := c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + fetchMeta := func(object string) (h http.Header) { + objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + h = make(http.Header) + for k, vs := range objInfo.Metadata { + if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { + for _, v := range vs { + h.Add(k, v) + } + } + } + return h + } + + // 1. create a client encrypted object to copy by uploading + const srcSize = 1024 * 1024 + buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB + metadata := make(http.Header) + metadata.Set("x-amz-meta-myheader", "myvalue") + m := make(map[string]string) + m["x-amz-meta-myheader"] = "myvalue" + _, err = c.PutObject(bucketName, "srcObject", + bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{UserMetadata: m}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectWithMetadata failed", err) + return + } + if !reflect.DeepEqual(metadata, fetchMeta("srcObject")) { + logError(testName, function, args, startTime, "", "Metadata match failed", err) + return + } + + // 2. create source + src := minio.NewSourceInfo(bucketName, "srcObject", nil) + // 2.1 create destination with metadata set + dst1, err := minio.NewDestinationInfo(bucketName, "dstObject-1", nil, map[string]string{"notmyheader": "notmyvalue"}) + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + + // 3. Check that copying to an object with metadata set resets + // the headers on the copy. + args["source"] = src + args["destination"] = dst1 + err = c.CopyObject(dst1, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) + return + } + + expectedHeaders := make(http.Header) + expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") + if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-1")) { + logError(testName, function, args, startTime, "", "Metadata match failed", err) + return + } + + // 4. create destination with no metadata set and same source + dst2, err := minio.NewDestinationInfo(bucketName, "dstObject-2", nil, nil) + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + src = minio.NewSourceInfo(bucketName, "srcObject", nil) + + // 5. Check that copying to an object with no metadata set, + // copies metadata. + args["source"] = src + args["destination"] = dst2 + err = c.CopyObject(dst2, src) + if err != nil { + logError(testName, function, args, startTime, "", "CopyObject failed", err) + return + } + + expectedHeaders = metadata + if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-2")) { + logError(testName, function, args, startTime, "", "Metadata match failed", err) + return + } + + // 6. Compose a pair of sources. + srcs := []minio.SourceInfo{ + minio.NewSourceInfo(bucketName, "srcObject", nil), + minio.NewSourceInfo(bucketName, "srcObject", nil), + } + dst3, err := minio.NewDestinationInfo(bucketName, "dstObject-3", nil, nil) + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + + function = "ComposeObject(destination, sources)" + args["source"] = srcs + args["destination"] = dst3 + err = c.ComposeObject(dst3, srcs) + if err != nil { + logError(testName, function, args, startTime, "", "ComposeObject failed", err) + return + } + + // Check that no headers are copied in this case + if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) { + logError(testName, function, args, startTime, "", "Metadata match failed", err) + return + } + + // 7. Compose a pair of sources with dest user metadata set. + srcs = []minio.SourceInfo{ + minio.NewSourceInfo(bucketName, "srcObject", nil), + minio.NewSourceInfo(bucketName, "srcObject", nil), + } + dst4, err := minio.NewDestinationInfo(bucketName, "dstObject-4", nil, map[string]string{"notmyheader": "notmyvalue"}) + if err != nil { + logError(testName, function, args, startTime, "", "NewDestinationInfo failed", err) + return + } + + function = "ComposeObject(destination, sources)" + args["source"] = srcs + args["destination"] = dst4 + err = c.ComposeObject(dst4, srcs) + if err != nil { + logError(testName, function, args, startTime, "", "ComposeObject failed", err) + return + } + + // Check that no headers are copied in this case + expectedHeaders = make(http.Header) + expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") + if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) { + logError(testName, function, args, startTime, "", "Metadata match failed", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +func testUserMetadataCopyingV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "CopyObject(destination, source)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + return + } + + // c.TraceOn(os.Stderr) + testUserMetadataCopyingWrapper(c) +} + +func testStorageClassMetadataPutObject() { + // initialize logging params + startTime := time.Now() + function := "testStorageClassMetadataPutObject()" + args := map[string]interface{}{} + testName := getFuncName() + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + return + } + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + fetchMeta := func(object string) (h http.Header) { + objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + h = make(http.Header) + for k, vs := range objInfo.Metadata { + if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { + for _, v := range vs { + h.Add(k, v) + } + } + } + return h + } + + metadata := make(http.Header) + metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + + emptyMetadata := make(http.Header) + + const srcSize = 1024 * 1024 + buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB + + _, err = c.PutObject(bucketName, "srcObjectRRSClass", + bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + // Get the returned metadata + returnedMeta := fetchMeta("srcObjectRRSClass") + + // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) + if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { + logError(testName, function, args, startTime, "", "Metadata match failed", err) + return + } + + metadata = make(http.Header) + metadata.Set("x-amz-storage-class", "STANDARD") + + _, err = c.PutObject(bucketName, "srcObjectSSClass", + bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClass")) { + logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) + return + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +func testStorageClassInvalidMetadataPutObject() { + // initialize logging params + startTime := time.Now() + function := "testStorageClassInvalidMetadataPutObject()" + args := map[string]interface{}{} + testName := getFuncName() + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + return + } + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + const srcSize = 1024 * 1024 + buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB + + _, err = c.PutObject(bucketName, "srcObjectRRSClass", + bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "INVALID_STORAGE_CLASS"}) + if err == nil { + logError(testName, function, args, startTime, "", "PutObject with invalid storage class passed, was expected to fail", err) + return + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +func testStorageClassMetadataCopyObject() { + // initialize logging params + startTime := time.Now() + function := "testStorageClassMetadataCopyObject()" + args := map[string]interface{}{} + testName := getFuncName() + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio v4 client object creation failed", err) + return + } + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + // Make a new bucket in 'us-east-1' (source bucket). + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + fetchMeta := func(object string) (h http.Header) { + objInfo, err := c.StatObject(bucketName, object, minio.StatObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + h = make(http.Header) + for k, vs := range objInfo.Metadata { + if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { + for _, v := range vs { + h.Add(k, v) + } + } + } + return h + } + + metadata := make(http.Header) + metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + + emptyMetadata := make(http.Header) + + const srcSize = 1024 * 1024 + buf := bytes.Repeat([]byte("abcde"), srcSize) + + // Put an object with RRS Storage class + _, err = c.PutObject(bucketName, "srcObjectRRSClass", + bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + // Make server side copy of object uploaded in previous step + src := minio.NewSourceInfo(bucketName, "srcObjectRRSClass", nil) + dst, err := minio.NewDestinationInfo(bucketName, "srcObjectRRSClassCopy", nil, nil) + c.CopyObject(dst, src) + + // Get the returned metadata + returnedMeta := fetchMeta("srcObjectRRSClassCopy") + + // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) + if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { + logError(testName, function, args, startTime, "", "Metadata match failed", err) + return + } + + metadata = make(http.Header) + metadata.Set("x-amz-storage-class", "STANDARD") + + // Put an object with Standard Storage class + _, err = c.PutObject(bucketName, "srcObjectSSClass", + bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + // Make server side copy of object uploaded in previous step + src = minio.NewSourceInfo(bucketName, "srcObjectSSClass", nil) + dst, err = minio.NewDestinationInfo(bucketName, "srcObjectSSClassCopy", nil, nil) + c.CopyObject(dst, src) + + // Fetch the meta data of copied object + if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClassCopy")) { + logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) + return + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + successLogger(testName, function, args, startTime).Info() +} + +// Test put object with size -1 byte object. +func testPutObjectNoLengthV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader, size, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "size": -1, + "opts": "", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + objectName := bucketName + "unique" + args["objectName"] = objectName + + bufSize := dataFileMap["datafile-65-MB"] + var reader = getDataReader("datafile-65-MB") + defer reader.Close() + args["size"] = bufSize + + // Upload an object. + n, err := c.PutObject(bucketName, objectName, reader, -1, minio.PutObjectOptions{}) + + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) + return + } + if n != int64(bufSize) { + logError(testName, function, args, startTime, "", "Expected upload object size "+string(bufSize)+" got "+string(n), err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test put objects of unknown size. +func testPutObjectsUnknownV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader,size,opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "size": "", + "opts": "", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Issues are revealed by trying to upload multiple files of unknown size + // sequentially (on 4GB machines) + for i := 1; i <= 4; i++ { + // Simulate that we could be receiving byte slices of data that we want + // to upload as a file + rpipe, wpipe := io.Pipe() + defer rpipe.Close() + go func() { + b := []byte("test") + wpipe.Write(b) + wpipe.Close() + }() + + // Upload the object. + objectName := fmt.Sprintf("%sunique%d", bucketName, i) + args["objectName"] = objectName + + n, err := c.PutObject(bucketName, objectName, rpipe, -1, minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) + return + } + args["size"] = n + if n != int64(4) { + logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(n), err) + return + } + + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test put object with 0 byte object. +func testPutObject0ByteV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader, size, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "size": 0, + "opts": "", + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + objectName := bucketName + "unique" + args["objectName"] = objectName + args["opts"] = minio.PutObjectOptions{} + + // Upload an object. + n, err := c.PutObject(bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{}) + + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) + return + } + if n != 0 { + logError(testName, function, args, startTime, "", "Expected upload object size 0 but got "+string(n), err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test expected error cases +func testComposeObjectErrorCases() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "ComposeObject(destination, sourceList)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + testComposeObjectErrorCasesWrapper(c) +} + +// Test concatenating 10K objects +func testCompose10KSources() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "ComposeObject(destination, sourceList)" + args := map[string]interface{}{} + + // Instantiate new minio client object + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client object creation failed", err) + return + } + + testComposeMultipleSources(c) +} + +// Tests comprehensive list of all methods. +func testFunctionalV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "testFunctionalV2()" + functionAll := "" + args := map[string]interface{}{} + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + return + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + location := "us-east-1" + // Make a new bucket. + function = "MakeBucket(bucketName, location)" + functionAll = "MakeBucket(bucketName, location)" + args = map[string]interface{}{ + "bucketName": bucketName, + "location": location, + } + err = c.MakeBucket(bucketName, location) + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + // Generate a random file name. + fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + file, err := os.Create(fileName) + if err != nil { + logError(testName, function, args, startTime, "", "file create failed", err) + return + } + for i := 0; i < 3; i++ { + buf := make([]byte, rand.Intn(1<<19)) + _, err = file.Write(buf) + if err != nil { + logError(testName, function, args, startTime, "", "file write failed", err) + return + } + } + file.Close() + + // Verify if bucket exits and you have access. + var exists bool + function = "BucketExists(bucketName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + } + exists, err = c.BucketExists(bucketName) + if err != nil { + logError(testName, function, args, startTime, "", "BucketExists failed", err) + return + } + if !exists { + logError(testName, function, args, startTime, "", "Could not find existing bucket "+bucketName, err) + return + } + + // Make the bucket 'public read/write'. + function = "SetBucketPolicy(bucketName, bucketPolicy)" + functionAll += ", " + function + + readWritePolicy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:ListBucketMultipartUploads", "s3:ListBucket"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::` + bucketName + `"],"Sid": ""}]}` + + args = map[string]interface{}{ + "bucketName": bucketName, + "bucketPolicy": readWritePolicy, + } + err = c.SetBucketPolicy(bucketName, readWritePolicy) + + if err != nil { + logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) + return + } + + // List all buckets. + function = "ListBuckets()" + functionAll += ", " + function + args = nil + buckets, err := c.ListBuckets() + if len(buckets) == 0 { + logError(testName, function, args, startTime, "", "List buckets cannot be empty", err) + return + } + if err != nil { + logError(testName, function, args, startTime, "", "ListBuckets failed", err) + return + } + + // Verify if previously created bucket is listed in list buckets. + bucketFound := false + for _, bucket := range buckets { + if bucket.Name == bucketName { + bucketFound = true + } + } + + // If bucket not found error out. + if !bucketFound { + logError(testName, function, args, startTime, "", "Bucket "+bucketName+"not found", err) + return + } + + objectName := bucketName + "unique" + + // Generate data + buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19)) + + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "contentType": "", + } + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + if n != int64(len(buf)) { + logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(n), err) + return + } + + objectNameNoLength := objectName + "-nolength" + args["objectName"] = objectNameNoLength + n, err = c.PutObject(bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + if n != int64(len(buf)) { + logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(n), err) + return + } + + // Instantiate a done channel to close all listing. + doneCh := make(chan struct{}) + defer close(doneCh) + + objFound := false + isRecursive := true // Recursive is true. + function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "isRecursive": isRecursive, + } + for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + logError(testName, function, args, startTime, "", "Could not find existing object "+objectName, err) + return + } + + incompObjNotFound := true + function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "isRecursive": isRecursive, + } + for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { + if objIncompl.Key != "" { + incompObjNotFound = false + break + } + } + if !incompObjNotFound { + logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err) + return + } + + function = "GetObject(bucketName, objectName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + } + newReader, err := c.GetObject(bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + newReadBytes, err := ioutil.ReadAll(newReader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + newReader.Close() + + if !bytes.Equal(newReadBytes, buf) { + logError(testName, function, args, startTime, "", "Bytes mismatch", err) + return + } + + function = "FGetObject(bucketName, objectName, fileName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "fileName": fileName + "-f", + } + err = c.FGetObject(bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "FgetObject failed", err) + return + } + + // Generate presigned HEAD object url. + function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "expires": 3600 * time.Second, + } + presignedHeadURL, err := c.PresignedHeadObject(bucketName, objectName, 3600*time.Second, nil) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) + return + } + // Verify if presigned url works. + resp, err := http.Head(presignedHeadURL.String()) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err) + return + } + if resp.StatusCode != http.StatusOK { + logError(testName, function, args, startTime, "", "PresignedHeadObject URL returns status "+string(resp.StatusCode), err) + return + } + if resp.Header.Get("ETag") == "" { + logError(testName, function, args, startTime, "", "Got empty ETag", err) + return + } + resp.Body.Close() + + // Generate presigned GET object url. + function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName, + "expires": 3600 * time.Second, + } + presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second, nil) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) + return + } + // Verify if presigned url works. + resp, err = http.Get(presignedGetURL.String()) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject URL GET request failed", err) + return + } + if resp.StatusCode != http.StatusOK { + logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) + return + } + newPresignedBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + resp.Body.Close() + if !bytes.Equal(newPresignedBytes, buf) { + logError(testName, function, args, startTime, "", "Bytes mismatch", err) + return + } + + // Set request parameters. + reqParams := make(url.Values) + reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") + // Generate presigned GET object url. + args["reqParams"] = reqParams + presignedGetURL, err = c.PresignedGetObject(bucketName, objectName, 3600*time.Second, reqParams) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) + return + } + // Verify if presigned url works. + resp, err = http.Get(presignedGetURL.String()) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedGetObject URL GET request failed", err) + return + } + if resp.StatusCode != http.StatusOK { + logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) + return + } + newPresignedBytes, err = ioutil.ReadAll(resp.Body) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + if !bytes.Equal(newPresignedBytes, buf) { + logError(testName, function, args, startTime, "", "Bytes mismatch", err) + return + } + // Verify content disposition. + if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { + logError(testName, function, args, startTime, "", "wrong Content-Disposition received ", err) + return + } + + function = "PresignedPutObject(bucketName, objectName, expires)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName + "-presigned", + "expires": 3600 * time.Second, + } + presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) + if err != nil { + logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) + return + } + + // Generate data more than 32K + buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024) + + req, err := http.NewRequest("PUT", presignedPutURL.String(), bytes.NewReader(buf)) + if err != nil { + logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) + return + } + httpClient := &http.Client{ + // Setting a sensible time out of 30secs to wait for response + // headers. Request is pro-actively cancelled after 30secs + // with no response. + Timeout: 30 * time.Second, + Transport: http.DefaultTransport, + } + resp, err = httpClient.Do(req) + if err != nil { + logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) + return + } + + function = "GetObject(bucketName, objectName)" + functionAll += ", " + function + args = map[string]interface{}{ + "bucketName": bucketName, + "objectName": objectName + "-presigned", + } + newReader, err = c.GetObject(bucketName, objectName+"-presigned", minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + newReadBytes, err = ioutil.ReadAll(newReader) + if err != nil { + logError(testName, function, args, startTime, "", "ReadAll failed", err) + return + } + newReader.Close() + + if !bytes.Equal(newReadBytes, buf) { + logError(testName, function, args, startTime, "", "Bytes mismatch", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + if err = os.Remove(fileName); err != nil { + logError(testName, function, args, startTime, "", "File remove failed", err) + return + } + if err = os.Remove(fileName + "-f"); err != nil { + logError(testName, function, args, startTime, "", "File removes failed", err) + return + } + successLogger(testName, functionAll, args, startTime).Info() +} + +// Test get object with GetObjectWithContext +func testGetObjectWithContext() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObjectWithContext(ctx, bucketName, objectName)" + args := map[string]interface{}{ + "ctx": "", + "bucketName": "", + "objectName": "", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + args["ctx"] = ctx + defer cancel() + + r, err := c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObjectWithContext failed unexpectedly", err) + return + } + + if _, err = r.Stat(); err == nil { + logError(testName, function, args, startTime, "", "GetObjectWithContext should fail on short timeout", err) + return + } + r.Close() + + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) + args["ctx"] = ctx + defer cancel() + + // Read the data back + r, err = c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObjectWithContext failed", err) + return + } + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "object Stat call failed", err) + return + } + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match: want "+string(bufSize)+", got"+string(st.Size), err) + return + } + if err := r.Close(); err != nil { + logError(testName, function, args, startTime, "", "object Close() call failed", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() + +} + +// Test get object with FGetObjectWithContext +func testFGetObjectWithContext() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "FGetObjectWithContext(ctx, bucketName, objectName, fileName)" + args := map[string]interface{}{ + "ctx": "", + "bucketName": "", + "objectName": "", + "fileName": "", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + bufSize := dataFileMap["datafile-1-MB"] + var reader = getDataReader("datafile-1-MB") + defer reader.Close() + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + args["ctx"] = ctx + defer cancel() + + fileName := "tempfile-context" + args["fileName"] = fileName + // Read the data back + err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) + if err == nil { + logError(testName, function, args, startTime, "", "FGetObjectWithContext should fail on short timeout", err) + return + } + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) + defer cancel() + + // Read the data back + err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "FGetObjectWithContext with long timeout failed", err) + return + } + if err = os.Remove(fileName + "-fcontext"); err != nil { + logError(testName, function, args, startTime, "", "Remove file failed", err) + return + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() + +} + +// Test get object ACLs with GetObjectACL +func testGetObjectACL() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObjectACL(bucketName, objectName)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // skipping region functional tests for non s3 runs + if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { + ignoredLog(testName, function, args, startTime, "Skipped region functional tests for non s3 runs").Info() + return + } + + // Instantiate new minio client object. + c, err := minio.NewV4( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + bufSize := dataFileMap["datafile-1-MB"] + var reader = getDataReader("datafile-1-MB") + defer reader.Close() + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Add meta data to add a canned acl + metaData := map[string]string{ + "X-Amz-Acl": "public-read-write", + } + + _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + // Read the data back + objectInfo, getObjectACLErr := c.GetObjectACL(bucketName, objectName) + if getObjectACLErr == nil { + logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr) + return + } + + s, ok := objectInfo.Metadata["X-Amz-Acl"] + if !ok { + logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Acl\"", nil) + return + } + + if len(s) != 1 { + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) + return + } + + if s[0] != "public-read-write" { + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil) + return + } + + bufSize = dataFileMap["datafile-1-MB"] + var reader2 = getDataReader("datafile-1-MB") + defer reader2.Close() + // Save the data + objectName = randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + // Add meta data to add a canned acl + metaData = map[string]string{ + "X-Amz-Grant-Read": "id=fooread@minio.go", + "X-Amz-Grant-Write": "id=foowrite@minio.go", + } + + _, err = c.PutObject(bucketName, objectName, reader2, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + // Read the data back + objectInfo, getObjectACLErr = c.GetObjectACL(bucketName, objectName) + if getObjectACLErr == nil { + logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr) + return + } + + if len(objectInfo.Metadata) != 3 { + logError(testName, function, args, startTime, "", "GetObjectACL fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil) + return + } + + s, ok = objectInfo.Metadata["X-Amz-Grant-Read"] + if !ok { + logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Read\"", nil) + return + } + + if len(s) != 1 { + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) + return + } + + if s[0] != "fooread@minio.go" { + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil) + return + } + + s, ok = objectInfo.Metadata["X-Amz-Grant-Write"] + if !ok { + logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Write\"", nil) + return + } + + if len(s) != 1 { + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) + return + } + + if s[0] != "foowrite@minio.go" { + logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() +} + +// Test validates putObject with context to see if request cancellation is honored for V2. +func testPutObjectWithContextV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObjectWithContext(ctx, bucketName, objectName, reader, size, opts)" + args := map[string]interface{}{ + "ctx": "", + "bucketName": "", + "objectName": "", + "size": "", + "opts": "", + } + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + defer c.RemoveBucket(bucketName) + bufSize := dataFileMap["datatfile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) + args["objectName"] = objectName + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + args["ctx"] = ctx + args["size"] = bufSize + defer cancel() + + _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectWithContext with short timeout failed", err) + return + } + + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) + args["ctx"] = ctx + + defer cancel() + reader = getDataReader("datafile-33-kB") + defer reader.Close() + _, err = c.PutObjectWithContext(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObjectWithContext with long timeout failed", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() + +} + +// Test get object with GetObjectWithContext +func testGetObjectWithContextV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "GetObjectWithContext(ctx, bucketName, objectName)" + args := map[string]interface{}{ + "ctx": "", + "bucketName": "", + "objectName": "", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + args["ctx"] = ctx + defer cancel() + + r, err := c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObjectWithContext failed unexpectedly", err) + return + } + if _, err = r.Stat(); err == nil { + logError(testName, function, args, startTime, "", "GetObjectWithContext should fail on short timeout", err) + return + } + r.Close() + + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) + defer cancel() + + // Read the data back + r, err = c.GetObjectWithContext(ctx, bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObjectWithContext shouldn't fail on longer timeout", err) + return + } + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "object Stat call failed", err) + return + } + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(bufSize)+" got "+string(st.Size), err) + return + } + if err := r.Close(); err != nil { + logError(testName, function, args, startTime, "", " object Close() call failed", err) + return + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() + +} + +// Test get object with FGetObjectWithContext +func testFGetObjectWithContextV2() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "FGetObjectWithContext(ctx, bucketName, objectName,fileName)" + args := map[string]interface{}{ + "ctx": "", + "bucketName": "", + "objectName": "", + "fileName": "", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v2 object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket call failed", err) + return + } + + bufSize := dataFileMap["datatfile-1-MB"] + var reader = getDataReader("datafile-1-MB") + defer reader.Close() + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + _, err = c.PutObject(bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject call failed", err) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + args["ctx"] = ctx + defer cancel() + + fileName := "tempfile-context" + args["fileName"] = fileName + + // Read the data back + err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) + if err == nil { + logError(testName, function, args, startTime, "", "FGetObjectWithContext should fail on short timeout", err) + return + } + ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) + defer cancel() + + // Read the data back + err = c.FGetObjectWithContext(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "FGetObjectWithContext call shouldn't fail on long timeout", err) + return + } + + if err = os.Remove(fileName + "-fcontext"); err != nil { + logError(testName, function, args, startTime, "", "Remove file failed", err) + return + } + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() + +} + +// Test list object v1 and V2 storage class fields +func testListObjects() { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "ListObjects(bucketName, objectPrefix, recursive, doneCh)" + args := map[string]interface{}{ + "bucketName": "", + "objectPrefix": "", + "recursive": "true", + } + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + os.Getenv(serverEndpoint), + os.Getenv(accessKey), + os.Getenv(secretKey), + mustParseBool(os.Getenv(enableHTTPS)), + ) + if err != nil { + logError(testName, function, args, startTime, "", "Minio client v4 object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + logError(testName, function, args, startTime, "", "MakeBucket failed", err) + return + } + + bufSize := dataFileMap["datafile-33-kB"] + var reader = getDataReader("datafile-33-kB") + defer reader.Close() + + // Save the data + objectName1 := randString(60, rand.NewSource(time.Now().UnixNano()), "") + + _, err = c.PutObject(bucketName, objectName1, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: "STANDARD"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject1 call failed", err) + return + } + + bufSize1 := dataFileMap["datafile-33-kB"] + var reader1 = getDataReader("datafile-33-kB") + defer reader1.Close() + objectName2 := randString(60, rand.NewSource(time.Now().UnixNano()), "") + + _, err = c.PutObject(bucketName, objectName2, reader1, int64(bufSize1), minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: "REDUCED_REDUNDANCY"}) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject2 call failed", err) + return + } + + // Create a done channel to control 'ListObjects' go routine. + doneCh := make(chan struct{}) + // Exit cleanly upon return. + defer close(doneCh) + + // check for storage-class from ListObjects result + for objInfo := range c.ListObjects(bucketName, "", true, doneCh) { + if objInfo.Err != nil { + logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err) + return + } + if objInfo.Key == objectName1 && objInfo.StorageClass != "STANDARD" { + // Ignored as Gateways (Azure/GCS etc) wont return storage class + ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info() + } + if objInfo.Key == objectName2 && objInfo.StorageClass != "REDUCED_REDUNDANCY" { + // Ignored as Gateways (Azure/GCS etc) wont return storage class + ignoredLog(testName, function, args, startTime, "ListObjects doesn't return expected storage class").Info() + } + } + + // check for storage-class from ListObjectsV2 result + for objInfo := range c.ListObjectsV2(bucketName, "", true, doneCh) { + if objInfo.Err != nil { + logError(testName, function, args, startTime, "", "ListObjectsV2 failed unexpectedly", err) + return + } + if objInfo.Key == objectName1 && objInfo.StorageClass != "STANDARD" { + // Ignored as Gateways (Azure/GCS etc) wont return storage class + ignoredLog(testName, function, args, startTime, "ListObjectsV2 doesn't return expected storage class").Info() + } + if objInfo.Key == objectName2 && objInfo.StorageClass != "REDUCED_REDUNDANCY" { + // Ignored as Gateways (Azure/GCS etc) wont return storage class + ignoredLog(testName, function, args, startTime, "ListObjectsV2 doesn't return expected storage class").Info() + } + } + + // Delete all objects and buckets + if err = cleanupBucket(bucketName, c); err != nil { + logError(testName, function, args, startTime, "", "Cleanup failed", err) + return + } + + successLogger(testName, function, args, startTime).Info() + +} + +// Convert string to bool and always return false if any error +func mustParseBool(str string) bool { + b, err := strconv.ParseBool(str) + if err != nil { + return false + } + return b +} + +func main() { + // Output to stdout instead of the default stderr + log.SetOutput(os.Stdout) + // create custom formatter + mintFormatter := mintJSONFormatter{} + // set custom formatter + log.SetFormatter(&mintFormatter) + // log Info or above -- success cases are Info level, failures are Fatal level + log.SetLevel(log.InfoLevel) + + tls := mustParseBool(os.Getenv(enableHTTPS)) + // execute tests + if isFullMode() { + testMakeBucketErrorV2() + testGetObjectClosedTwiceV2() + testFPutObjectV2() + testMakeBucketRegionsV2() + testGetObjectReadSeekFunctionalV2() + testGetObjectReadAtFunctionalV2() + testCopyObjectV2() + testFunctionalV2() + testComposeObjectErrorCasesV2() + testCompose10KSourcesV2() + testUserMetadataCopyingV2() + testPutObject0ByteV2() + testPutObjectNoLengthV2() + testPutObjectsUnknownV2() + testGetObjectWithContextV2() + testFPutObjectWithContextV2() + testFGetObjectWithContextV2() + testPutObjectWithContextV2() + testMakeBucketError() + testMakeBucketRegions() + testPutObjectWithMetadata() + testPutObjectReadAt() + testPutObjectStreaming() + testGetObjectSeekEnd() + testGetObjectClosedTwice() + testRemoveMultipleObjects() + testFPutObjectMultipart() + testFPutObject() + testGetObjectReadSeekFunctional() + testGetObjectReadAtFunctional() + testPresignedPostPolicy() + testCopyObject() + testComposeObjectErrorCases() + testCompose10KSources() + testUserMetadataCopying() + testBucketNotification() + testFunctional() + testGetObjectModified() + testPutObjectUploadSeekedObject() + testGetObjectWithContext() + testFPutObjectWithContext() + testFGetObjectWithContext() + testGetObjectACL() + testPutObjectWithContext() + testStorageClassMetadataPutObject() + testStorageClassInvalidMetadataPutObject() + testStorageClassMetadataCopyObject() + testPutObjectWithContentLanguage() + testListObjects() + + // SSE-C tests will only work over TLS connection. + if tls { + testEncryptionPutGet() + testEncryptionFPut() + testEncryptedGetObjectReadAtFunctional() + testEncryptedGetObjectReadSeekFunctional() + testEncryptedCopyObjectV2() + testEncryptedCopyObject() + testEncryptedEmptyObject() + testDecryptedCopyObject() + testCoreEncryptedCopyObjectPart() + } + } else { + testFunctional() + testFunctionalV2() + } +} diff --git a/get-options_test.go b/get-options_test.go new file mode 100644 index 0000000..c5344a0 --- /dev/null +++ b/get-options_test.go @@ -0,0 +1,57 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "fmt" + "testing" +) + +func TestSetHeader(t *testing.T) { + testCases := []struct { + start int64 + end int64 + errVal error + expected string + }{ + {0, 10, nil, "bytes=0-10"}, + {1, 10, nil, "bytes=1-10"}, + {5, 0, nil, "bytes=5-"}, + {0, -5, nil, "bytes=-5"}, + {0, 0, nil, "bytes=0-0"}, + {11, 10, fmt.Errorf("Invalid range specified: start=11 end=10"), + ""}, + {-1, 10, fmt.Errorf("Invalid range specified: start=-1 end=10"), ""}, + {-1, 0, fmt.Errorf("Invalid range specified: start=-1 end=0"), ""}, + {1, -5, fmt.Errorf("Invalid range specified: start=1 end=-5"), ""}, + } + for i, testCase := range testCases { + opts := GetObjectOptions{} + err := opts.SetRange(testCase.start, testCase.end) + if err == nil && testCase.errVal != nil { + t.Errorf("Test %d: Expected to fail with '%v' but it passed", + i+1, testCase.errVal) + } else if err != nil && testCase.errVal.Error() != err.Error() { + t.Errorf("Test %d: Expected error '%v' but got error '%v'", + i+1, testCase.errVal, err) + } else if err == nil && opts.headers["Range"] != testCase.expected { + t.Errorf("Test %d: Expected range header '%s', but got '%s'", + i+1, testCase.expected, opts.headers["Range"]) + } + } +} diff --git a/hook-reader.go b/hook-reader.go new file mode 100644 index 0000000..8f32291 --- /dev/null +++ b/hook-reader.go @@ -0,0 +1,71 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import "io" + +// hookReader hooks additional reader in the source stream. It is +// useful for making progress bars. Second reader is appropriately +// notified about the exact number of bytes read from the primary +// source on each Read operation. +type hookReader struct { + source io.Reader + hook io.Reader +} + +// Seek implements io.Seeker. Seeks source first, and if necessary +// seeks hook if Seek method is appropriately found. +func (hr *hookReader) Seek(offset int64, whence int) (n int64, err error) { + // Verify for source has embedded Seeker, use it. + sourceSeeker, ok := hr.source.(io.Seeker) + if ok { + return sourceSeeker.Seek(offset, whence) + } + // Verify if hook has embedded Seeker, use it. + hookSeeker, ok := hr.hook.(io.Seeker) + if ok { + return hookSeeker.Seek(offset, whence) + } + return n, nil +} + +// Read implements io.Reader. Always reads from the source, the return +// value 'n' number of bytes are reported through the hook. Returns +// error for all non io.EOF conditions. +func (hr *hookReader) Read(b []byte) (n int, err error) { + n, err = hr.source.Read(b) + if err != nil && err != io.EOF { + return n, err + } + // Progress the hook with the total read bytes from the source. + if _, herr := hr.hook.Read(b[:n]); herr != nil { + if herr != io.EOF { + return n, herr + } + } + return n, err +} + +// newHook returns a io.ReadSeeker which implements hookReader that +// reports the data read from the source to the hook. +func newHook(source, hook io.Reader) io.Reader { + if hook == nil { + return source + } + return &hookReader{source, hook} +} diff --git a/pkg/credentials/chain.go b/pkg/credentials/chain.go new file mode 100644 index 0000000..e29826f --- /dev/null +++ b/pkg/credentials/chain.go @@ -0,0 +1,89 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +// A Chain will search for a provider which returns credentials +// and cache that provider until Retrieve is called again. +// +// The Chain provides a way of chaining multiple providers together +// which will pick the first available using priority order of the +// Providers in the list. +// +// If none of the Providers retrieve valid credentials Value, ChainProvider's +// Retrieve() will return the no credentials value. +// +// If a Provider is found which returns valid credentials Value ChainProvider +// will cache that Provider for all calls to IsExpired(), until Retrieve is +// called again after IsExpired() is true. +// +// creds := credentials.NewChainCredentials( +// []credentials.Provider{ +// &credentials.EnvAWSS3{}, +// &credentials.EnvMinio{}, +// }) +// +// // Usage of ChainCredentials. +// mc, err := minio.NewWithCredentials(endpoint, creds, secure, "us-east-1") +// if err != nil { +// log.Fatalln(err) +// } +// +type Chain struct { + Providers []Provider + curr Provider +} + +// NewChainCredentials returns a pointer to a new Credentials object +// wrapping a chain of providers. +func NewChainCredentials(providers []Provider) *Credentials { + return New(&Chain{ + Providers: append([]Provider{}, providers...), + }) +} + +// Retrieve returns the credentials value, returns no credentials(anonymous) +// if no credentials provider returned any value. +// +// If a provider is found with credentials, it will be cached and any calls +// to IsExpired() will return the expired state of the cached provider. +func (c *Chain) Retrieve() (Value, error) { + for _, p := range c.Providers { + creds, _ := p.Retrieve() + // Always prioritize non-anonymous providers, if any. + if creds.AccessKeyID == "" && creds.SecretAccessKey == "" { + continue + } + c.curr = p + return creds, nil + } + // At this point we have exhausted all the providers and + // are left without any credentials return anonymous. + return Value{ + SignerType: SignatureAnonymous, + }, nil +} + +// IsExpired will returned the expired state of the currently cached provider +// if there is one. If there is no current provider, true will be returned. +func (c *Chain) IsExpired() bool { + if c.curr != nil { + return c.curr.IsExpired() + } + + return true +} diff --git a/pkg/credentials/chain_test.go b/pkg/credentials/chain_test.go new file mode 100644 index 0000000..d26e376 --- /dev/null +++ b/pkg/credentials/chain_test.go @@ -0,0 +1,144 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "errors" + "testing" +) + +type testCredProvider struct { + creds Value + expired bool + err error +} + +func (s *testCredProvider) Retrieve() (Value, error) { + s.expired = false + return s.creds, s.err +} +func (s *testCredProvider) IsExpired() bool { + return s.expired +} + +func TestChainGet(t *testing.T) { + p := &Chain{ + Providers: []Provider{ + &credProvider{err: errors.New("FirstError")}, + &credProvider{err: errors.New("SecondError")}, + &testCredProvider{ + creds: Value{ + AccessKeyID: "AKIF", + SecretAccessKey: "NOSECRET", + SessionToken: "", + }, + }, + &credProvider{ + creds: Value{ + AccessKeyID: "AKID", + SecretAccessKey: "SECRET", + SessionToken: "", + }, + }, + }, + } + + creds, err := p.Retrieve() + if err != nil { + t.Fatal(err) + } + + // Also check credentials + if creds.AccessKeyID != "AKIF" { + t.Fatalf("Expected 'AKIF', got %s", creds.AccessKeyID) + } + if creds.SecretAccessKey != "NOSECRET" { + t.Fatalf("Expected 'NOSECRET', got %s", creds.SecretAccessKey) + } + if creds.SessionToken != "" { + t.Fatalf("Expected empty token, got %s", creds.SessionToken) + } +} + +func TestChainIsExpired(t *testing.T) { + credProvider := &credProvider{ + creds: Value{ + AccessKeyID: "UXHW", + SecretAccessKey: "MYSECRET", + SessionToken: "", + }, + expired: true, + } + p := &Chain{ + Providers: []Provider{ + credProvider, + }, + } + + if !p.IsExpired() { + t.Fatal("Expected expired to be true before any Retrieve") + } + + _, err := p.Retrieve() + if err != nil { + t.Fatal(err) + } + + if p.IsExpired() { + t.Fatal("Expected to be not expired after Retrieve") + } +} + +func TestChainWithNoProvider(t *testing.T) { + p := &Chain{ + Providers: []Provider{}, + } + if !p.IsExpired() { + t.Fatal("Expected to be expired with no providers") + } + _, err := p.Retrieve() + if err != nil { + if err.Error() != "No valid providers found []" { + t.Error(err) + } + } +} + +func TestChainProviderWithNoValidProvider(t *testing.T) { + errs := []error{ + errors.New("FirstError"), + errors.New("SecondError"), + } + p := &Chain{ + Providers: []Provider{ + &credProvider{err: errs[0]}, + &credProvider{err: errs[1]}, + }, + } + + if !p.IsExpired() { + t.Fatal("Expected to be expired with no providers") + } + + _, err := p.Retrieve() + if err != nil { + if err.Error() != "No valid providers found [FirstError SecondError]" { + t.Error(err) + } + } +} diff --git a/pkg/credentials/config.json.sample b/pkg/credentials/config.json.sample new file mode 100644 index 0000000..130746f --- /dev/null +++ b/pkg/credentials/config.json.sample @@ -0,0 +1,17 @@ +{ + "version": "8", + "hosts": { + "play": { + "url": "https://play.minio.io:9000", + "accessKey": "Q3AM3UQ867SPQQA43P2F", + "secretKey": "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", + "api": "S3v2" + }, + "s3": { + "url": "https://s3.amazonaws.com", + "accessKey": "accessKey", + "secretKey": "secret", + "api": "S3v4" + } + } +} \ No newline at end of file diff --git a/pkg/credentials/credentials.go b/pkg/credentials/credentials.go new file mode 100644 index 0000000..4bfdad4 --- /dev/null +++ b/pkg/credentials/credentials.go @@ -0,0 +1,175 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "sync" + "time" +) + +// A Value is the AWS credentials value for individual credential fields. +type Value struct { + // AWS Access key ID + AccessKeyID string + + // AWS Secret Access Key + SecretAccessKey string + + // AWS Session Token + SessionToken string + + // Signature Type. + SignerType SignatureType +} + +// A Provider is the interface for any component which will provide credentials +// Value. A provider is required to manage its own Expired state, and what to +// be expired means. +type Provider interface { + // Retrieve returns nil if it successfully retrieved the value. + // Error is returned if the value were not obtainable, or empty. + Retrieve() (Value, error) + + // IsExpired returns if the credentials are no longer valid, and need + // to be retrieved. + IsExpired() bool +} + +// A Expiry provides shared expiration logic to be used by credentials +// providers to implement expiry functionality. +// +// The best method to use this struct is as an anonymous field within the +// provider's struct. +// +// Example: +// type IAMCredentialProvider struct { +// Expiry +// ... +// } +type Expiry struct { + // The date/time when to expire on + expiration time.Time + + // If set will be used by IsExpired to determine the current time. + // Defaults to time.Now if CurrentTime is not set. + CurrentTime func() time.Time +} + +// SetExpiration sets the expiration IsExpired will check when called. +// +// If window is greater than 0 the expiration time will be reduced by the +// window value. +// +// Using a window is helpful to trigger credentials to expire sooner than +// the expiration time given to ensure no requests are made with expired +// tokens. +func (e *Expiry) SetExpiration(expiration time.Time, window time.Duration) { + e.expiration = expiration + if window > 0 { + e.expiration = e.expiration.Add(-window) + } +} + +// IsExpired returns if the credentials are expired. +func (e *Expiry) IsExpired() bool { + if e.CurrentTime == nil { + e.CurrentTime = time.Now + } + return e.expiration.Before(e.CurrentTime()) +} + +// Credentials - A container for synchronous safe retrieval of credentials Value. +// Credentials will cache the credentials value until they expire. Once the value +// expires the next Get will attempt to retrieve valid credentials. +// +// Credentials is safe to use across multiple goroutines and will manage the +// synchronous state so the Providers do not need to implement their own +// synchronization. +// +// The first Credentials.Get() will always call Provider.Retrieve() to get the +// first instance of the credentials Value. All calls to Get() after that +// will return the cached credentials Value until IsExpired() returns true. +type Credentials struct { + sync.Mutex + + creds Value + forceRefresh bool + provider Provider +} + +// New returns a pointer to a new Credentials with the provider set. +func New(provider Provider) *Credentials { + return &Credentials{ + provider: provider, + forceRefresh: true, + } +} + +// Get returns the credentials value, or error if the credentials Value failed +// to be retrieved. +// +// Will return the cached credentials Value if it has not expired. If the +// credentials Value has expired the Provider's Retrieve() will be called +// to refresh the credentials. +// +// If Credentials.Expire() was called the credentials Value will be force +// expired, and the next call to Get() will cause them to be refreshed. +func (c *Credentials) Get() (Value, error) { + c.Lock() + defer c.Unlock() + + if c.isExpired() { + creds, err := c.provider.Retrieve() + if err != nil { + return Value{}, err + } + c.creds = creds + c.forceRefresh = false + } + + return c.creds, nil +} + +// Expire expires the credentials and forces them to be retrieved on the +// next call to Get(). +// +// This will override the Provider's expired state, and force Credentials +// to call the Provider's Retrieve(). +func (c *Credentials) Expire() { + c.Lock() + defer c.Unlock() + + c.forceRefresh = true +} + +// IsExpired returns if the credentials are no longer valid, and need +// to be refreshed. +// +// If the Credentials were forced to be expired with Expire() this will +// reflect that override. +func (c *Credentials) IsExpired() bool { + c.Lock() + defer c.Unlock() + + return c.isExpired() +} + +// isExpired helper method wrapping the definition of expired credentials. +func (c *Credentials) isExpired() bool { + return c.forceRefresh || c.provider.IsExpired() +} diff --git a/pkg/credentials/credentials.sample b/pkg/credentials/credentials.sample new file mode 100644 index 0000000..7fc91d9 --- /dev/null +++ b/pkg/credentials/credentials.sample @@ -0,0 +1,12 @@ +[default] +aws_access_key_id = accessKey +aws_secret_access_key = secret +aws_session_token = token + +[no_token] +aws_access_key_id = accessKey +aws_secret_access_key = secret + +[with_colon] +aws_access_key_id: accessKey +aws_secret_access_key: secret diff --git a/pkg/credentials/credentials_test.go b/pkg/credentials/credentials_test.go new file mode 100644 index 0000000..92c77c4 --- /dev/null +++ b/pkg/credentials/credentials_test.go @@ -0,0 +1,73 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "errors" + "testing" +) + +type credProvider struct { + creds Value + expired bool + err error +} + +func (s *credProvider) Retrieve() (Value, error) { + s.expired = false + return s.creds, s.err +} +func (s *credProvider) IsExpired() bool { + return s.expired +} + +func TestCredentialsGet(t *testing.T) { + c := New(&credProvider{ + creds: Value{ + AccessKeyID: "UXHW", + SecretAccessKey: "MYSECRET", + SessionToken: "", + }, + expired: true, + }) + + creds, err := c.Get() + if err != nil { + t.Fatal(err) + } + if "UXHW" != creds.AccessKeyID { + t.Errorf("Expected \"UXHW\", got %s", creds.AccessKeyID) + } + if "MYSECRET" != creds.SecretAccessKey { + t.Errorf("Expected \"MYSECRET\", got %s", creds.SecretAccessKey) + } + if creds.SessionToken != "" { + t.Errorf("Expected session token to be empty, got %s", creds.SessionToken) + } +} + +func TestCredentialsGetWithError(t *testing.T) { + c := New(&credProvider{err: errors.New("Custom error")}) + + _, err := c.Get() + if err != nil { + if err.Error() != "Custom error" { + t.Errorf("Expected \"Custom error\", got %s", err.Error()) + } + } +} diff --git a/pkg/credentials/doc.go b/pkg/credentials/doc.go new file mode 100644 index 0000000..c48784b --- /dev/null +++ b/pkg/credentials/doc.go @@ -0,0 +1,62 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package credentials provides credential retrieval and management +// for S3 compatible object storage. +// +// By default the Credentials.Get() will cache the successful result of a +// Provider's Retrieve() until Provider.IsExpired() returns true. At which +// point Credentials will call Provider's Retrieve() to get new credential Value. +// +// The Provider is responsible for determining when credentials have expired. +// It is also important to note that Credentials will always call Retrieve the +// first time Credentials.Get() is called. +// +// Example of using the environment variable credentials. +// +// creds := NewFromEnv() +// // Retrieve the credentials value +// credValue, err := creds.Get() +// if err != nil { +// // handle error +// } +// +// Example of forcing credentials to expire and be refreshed on the next Get(). +// This may be helpful to proactively expire credentials and refresh them sooner +// than they would naturally expire on their own. +// +// creds := NewFromIAM("") +// creds.Expire() +// credsValue, err := creds.Get() +// // New credentials will be retrieved instead of from cache. +// +// +// Custom Provider +// +// Each Provider built into this package also provides a helper method to generate +// a Credentials pointer setup with the provider. To use a custom Provider just +// create a type which satisfies the Provider interface and pass it to the +// NewCredentials method. +// +// type MyProvider struct{} +// func (m *MyProvider) Retrieve() (Value, error) {...} +// func (m *MyProvider) IsExpired() bool {...} +// +// creds := NewCredentials(&MyProvider{}) +// credValue, err := creds.Get() +// +package credentials diff --git a/pkg/credentials/env_aws.go b/pkg/credentials/env_aws.go new file mode 100644 index 0000000..f9b2cc3 --- /dev/null +++ b/pkg/credentials/env_aws.go @@ -0,0 +1,71 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import "os" + +// A EnvAWS retrieves credentials from the environment variables of the +// running process. EnvAWSironment credentials never expire. +// +// EnvAWSironment variables used: +// +// * Access Key ID: AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY. +// * Secret Access Key: AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY. +// * Secret Token: AWS_SESSION_TOKEN. +type EnvAWS struct { + retrieved bool +} + +// NewEnvAWS returns a pointer to a new Credentials object +// wrapping the environment variable provider. +func NewEnvAWS() *Credentials { + return New(&EnvAWS{}) +} + +// Retrieve retrieves the keys from the environment. +func (e *EnvAWS) Retrieve() (Value, error) { + e.retrieved = false + + id := os.Getenv("AWS_ACCESS_KEY_ID") + if id == "" { + id = os.Getenv("AWS_ACCESS_KEY") + } + + secret := os.Getenv("AWS_SECRET_ACCESS_KEY") + if secret == "" { + secret = os.Getenv("AWS_SECRET_KEY") + } + + signerType := SignatureV4 + if id == "" || secret == "" { + signerType = SignatureAnonymous + } + + e.retrieved = true + return Value{ + AccessKeyID: id, + SecretAccessKey: secret, + SessionToken: os.Getenv("AWS_SESSION_TOKEN"), + SignerType: signerType, + }, nil +} + +// IsExpired returns if the credentials have been retrieved. +func (e *EnvAWS) IsExpired() bool { + return !e.retrieved +} diff --git a/pkg/credentials/env_minio.go b/pkg/credentials/env_minio.go new file mode 100644 index 0000000..d72e771 --- /dev/null +++ b/pkg/credentials/env_minio.go @@ -0,0 +1,62 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import "os" + +// A EnvMinio retrieves credentials from the environment variables of the +// running process. EnvMinioironment credentials never expire. +// +// EnvMinioironment variables used: +// +// * Access Key ID: MINIO_ACCESS_KEY. +// * Secret Access Key: MINIO_SECRET_KEY. +type EnvMinio struct { + retrieved bool +} + +// NewEnvMinio returns a pointer to a new Credentials object +// wrapping the environment variable provider. +func NewEnvMinio() *Credentials { + return New(&EnvMinio{}) +} + +// Retrieve retrieves the keys from the environment. +func (e *EnvMinio) Retrieve() (Value, error) { + e.retrieved = false + + id := os.Getenv("MINIO_ACCESS_KEY") + secret := os.Getenv("MINIO_SECRET_KEY") + + signerType := SignatureV4 + if id == "" || secret == "" { + signerType = SignatureAnonymous + } + + e.retrieved = true + return Value{ + AccessKeyID: id, + SecretAccessKey: secret, + SignerType: signerType, + }, nil +} + +// IsExpired returns if the credentials have been retrieved. +func (e *EnvMinio) IsExpired() bool { + return !e.retrieved +} diff --git a/pkg/credentials/env_test.go b/pkg/credentials/env_test.go new file mode 100644 index 0000000..09cd77f --- /dev/null +++ b/pkg/credentials/env_test.go @@ -0,0 +1,105 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "os" + "reflect" + "testing" +) + +func TestEnvAWSRetrieve(t *testing.T) { + os.Clearenv() + os.Setenv("AWS_ACCESS_KEY_ID", "access") + os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") + os.Setenv("AWS_SESSION_TOKEN", "token") + + e := EnvAWS{} + if !e.IsExpired() { + t.Error("Expect creds to be expired before retrieve.") + } + + creds, err := e.Retrieve() + if err != nil { + t.Fatal(err) + } + + expectedCreds := Value{ + AccessKeyID: "access", + SecretAccessKey: "secret", + SessionToken: "token", + SignerType: SignatureV4, + } + if !reflect.DeepEqual(creds, expectedCreds) { + t.Errorf("Expected %v, got %v", expectedCreds, creds) + } + + if e.IsExpired() { + t.Error("Expect creds to not be expired after retrieve.") + } + + os.Clearenv() + os.Setenv("AWS_ACCESS_KEY", "access") + os.Setenv("AWS_SECRET_KEY", "secret") + + expectedCreds = Value{ + AccessKeyID: "access", + SecretAccessKey: "secret", + SignerType: SignatureV4, + } + + creds, err = e.Retrieve() + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(creds, expectedCreds) { + t.Errorf("Expected %v, got %v", expectedCreds, creds) + } + +} + +func TestEnvMinioRetrieve(t *testing.T) { + os.Clearenv() + + os.Setenv("MINIO_ACCESS_KEY", "access") + os.Setenv("MINIO_SECRET_KEY", "secret") + + e := EnvMinio{} + if !e.IsExpired() { + t.Error("Expect creds to be expired before retrieve.") + } + + creds, err := e.Retrieve() + if err != nil { + t.Fatal(err) + } + + expectedCreds := Value{ + AccessKeyID: "access", + SecretAccessKey: "secret", + SignerType: SignatureV4, + } + if !reflect.DeepEqual(creds, expectedCreds) { + t.Errorf("Expected %v, got %v", expectedCreds, creds) + } + + if e.IsExpired() { + t.Error("Expect creds to not be expired after retrieve.") + } +} diff --git a/pkg/credentials/file_aws_credentials.go b/pkg/credentials/file_aws_credentials.go new file mode 100644 index 0000000..5ad6830 --- /dev/null +++ b/pkg/credentials/file_aws_credentials.go @@ -0,0 +1,120 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "os" + "path/filepath" + + "github.com/go-ini/ini" + homedir "github.com/mitchellh/go-homedir" +) + +// A FileAWSCredentials retrieves credentials from the current user's home +// directory, and keeps track if those credentials are expired. +// +// Profile ini file example: $HOME/.aws/credentials +type FileAWSCredentials struct { + // Path to the shared credentials file. + // + // If empty will look for "AWS_SHARED_CREDENTIALS_FILE" env variable. If the + // env value is empty will default to current user's home directory. + // Linux/OSX: "$HOME/.aws/credentials" + // Windows: "%USERPROFILE%\.aws\credentials" + filename string + + // AWS Profile to extract credentials from the shared credentials file. If empty + // will default to environment variable "AWS_PROFILE" or "default" if + // environment variable is also not set. + profile string + + // retrieved states if the credentials have been successfully retrieved. + retrieved bool +} + +// NewFileAWSCredentials returns a pointer to a new Credentials object +// wrapping the Profile file provider. +func NewFileAWSCredentials(filename string, profile string) *Credentials { + return New(&FileAWSCredentials{ + filename: filename, + profile: profile, + }) +} + +// Retrieve reads and extracts the shared credentials from the current +// users home directory. +func (p *FileAWSCredentials) Retrieve() (Value, error) { + if p.filename == "" { + p.filename = os.Getenv("AWS_SHARED_CREDENTIALS_FILE") + if p.filename == "" { + homeDir, err := homedir.Dir() + if err != nil { + return Value{}, err + } + p.filename = filepath.Join(homeDir, ".aws", "credentials") + } + } + if p.profile == "" { + p.profile = os.Getenv("AWS_PROFILE") + if p.profile == "" { + p.profile = "default" + } + } + + p.retrieved = false + + iniProfile, err := loadProfile(p.filename, p.profile) + if err != nil { + return Value{}, err + } + + // Default to empty string if not found. + id := iniProfile.Key("aws_access_key_id") + // Default to empty string if not found. + secret := iniProfile.Key("aws_secret_access_key") + // Default to empty string if not found. + token := iniProfile.Key("aws_session_token") + + p.retrieved = true + return Value{ + AccessKeyID: id.String(), + SecretAccessKey: secret.String(), + SessionToken: token.String(), + SignerType: SignatureV4, + }, nil +} + +// IsExpired returns if the shared credentials have expired. +func (p *FileAWSCredentials) IsExpired() bool { + return !p.retrieved +} + +// loadProfiles loads from the file pointed to by shared credentials filename for profile. +// The credentials retrieved from the profile will be returned or error. Error will be +// returned if it fails to read from the file, or the data is invalid. +func loadProfile(filename, profile string) (*ini.Section, error) { + config, err := ini.Load(filename) + if err != nil { + return nil, err + } + iniProfile, err := config.GetSection(profile) + if err != nil { + return nil, err + } + return iniProfile, nil +} diff --git a/pkg/credentials/file_minio_client.go b/pkg/credentials/file_minio_client.go new file mode 100644 index 0000000..6a6827e --- /dev/null +++ b/pkg/credentials/file_minio_client.go @@ -0,0 +1,133 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "runtime" + + homedir "github.com/mitchellh/go-homedir" +) + +// A FileMinioClient retrieves credentials from the current user's home +// directory, and keeps track if those credentials are expired. +// +// Configuration file example: $HOME/.mc/config.json +type FileMinioClient struct { + // Path to the shared credentials file. + // + // If empty will look for "MINIO_SHARED_CREDENTIALS_FILE" env variable. If the + // env value is empty will default to current user's home directory. + // Linux/OSX: "$HOME/.mc/config.json" + // Windows: "%USERALIAS%\mc\config.json" + filename string + + // Minio Alias to extract credentials from the shared credentials file. If empty + // will default to environment variable "MINIO_ALIAS" or "default" if + // environment variable is also not set. + alias string + + // retrieved states if the credentials have been successfully retrieved. + retrieved bool +} + +// NewFileMinioClient returns a pointer to a new Credentials object +// wrapping the Alias file provider. +func NewFileMinioClient(filename string, alias string) *Credentials { + return New(&FileMinioClient{ + filename: filename, + alias: alias, + }) +} + +// Retrieve reads and extracts the shared credentials from the current +// users home directory. +func (p *FileMinioClient) Retrieve() (Value, error) { + if p.filename == "" { + if value, ok := os.LookupEnv("MINIO_SHARED_CREDENTIALS_FILE"); ok { + p.filename = value + } else { + homeDir, err := homedir.Dir() + if err != nil { + return Value{}, err + } + p.filename = filepath.Join(homeDir, ".mc", "config.json") + if runtime.GOOS == "windows" { + p.filename = filepath.Join(homeDir, "mc", "config.json") + } + } + } + + if p.alias == "" { + p.alias = os.Getenv("MINIO_ALIAS") + if p.alias == "" { + p.alias = "s3" + } + } + + p.retrieved = false + + hostCfg, err := loadAlias(p.filename, p.alias) + if err != nil { + return Value{}, err + } + + p.retrieved = true + return Value{ + AccessKeyID: hostCfg.AccessKey, + SecretAccessKey: hostCfg.SecretKey, + SignerType: parseSignatureType(hostCfg.API), + }, nil +} + +// IsExpired returns if the shared credentials have expired. +func (p *FileMinioClient) IsExpired() bool { + return !p.retrieved +} + +// hostConfig configuration of a host. +type hostConfig struct { + URL string `json:"url"` + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` + API string `json:"api"` +} + +// config config version. +type config struct { + Version string `json:"version"` + Hosts map[string]hostConfig `json:"hosts"` +} + +// loadAliass loads from the file pointed to by shared credentials filename for alias. +// The credentials retrieved from the alias will be returned or error. Error will be +// returned if it fails to read from the file. +func loadAlias(filename, alias string) (hostConfig, error) { + cfg := &config{} + configBytes, err := ioutil.ReadFile(filename) + if err != nil { + return hostConfig{}, err + } + if err = json.Unmarshal(configBytes, cfg); err != nil { + return hostConfig{}, err + } + return cfg.Hosts[alias], nil +} diff --git a/pkg/credentials/file_test.go b/pkg/credentials/file_test.go new file mode 100644 index 0000000..c85c104 --- /dev/null +++ b/pkg/credentials/file_test.go @@ -0,0 +1,189 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "os" + "path/filepath" + "testing" +) + +func TestFileAWS(t *testing.T) { + os.Clearenv() + + creds := NewFileAWSCredentials("credentials.sample", "") + credValues, err := creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "accessKey" { + t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "secret" { + t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) + } + if credValues.SessionToken != "token" { + t.Errorf("Expected 'token', got %s'", credValues.SessionToken) + } + + os.Setenv("AWS_SHARED_CREDENTIALS_FILE", "credentials.sample") + creds = NewFileAWSCredentials("", "") + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "accessKey" { + t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "secret" { + t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) + } + if credValues.SessionToken != "token" { + t.Errorf("Expected 'token', got %s'", credValues.SessionToken) + } + + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + os.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(wd, "credentials.sample")) + creds = NewFileAWSCredentials("", "") + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "accessKey" { + t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "secret" { + t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) + } + if credValues.SessionToken != "token" { + t.Errorf("Expected 'token', got %s'", credValues.SessionToken) + } + + os.Clearenv() + os.Setenv("AWS_PROFILE", "no_token") + + creds = NewFileAWSCredentials("credentials.sample", "") + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "accessKey" { + t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "secret" { + t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) + } + + os.Clearenv() + + creds = NewFileAWSCredentials("credentials.sample", "no_token") + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "accessKey" { + t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "secret" { + t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) + } + + creds = NewFileAWSCredentials("credentials-non-existent.sample", "no_token") + _, err = creds.Get() + if !os.IsNotExist(err) { + t.Errorf("Expected open non-existent.json: no such file or directory, got %s", err) + } + if !creds.IsExpired() { + t.Error("Should be expired if not loaded") + } +} + +func TestFileMinioClient(t *testing.T) { + os.Clearenv() + + creds := NewFileMinioClient("config.json.sample", "") + credValues, err := creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "accessKey" { + t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "secret" { + t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) + } + if credValues.SignerType != SignatureV4 { + t.Errorf("Expected 'S3v4', got %s'", credValues.SignerType) + } + + os.Clearenv() + os.Setenv("MINIO_ALIAS", "play") + + creds = NewFileMinioClient("config.json.sample", "") + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "Q3AM3UQ867SPQQA43P2F" { + t.Errorf("Expected 'Q3AM3UQ867SPQQA43P2F', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" { + t.Errorf("Expected 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG', got %s'", credValues.SecretAccessKey) + } + if credValues.SignerType != SignatureV2 { + t.Errorf("Expected 'S3v2', got %s'", credValues.SignerType) + } + + os.Clearenv() + + creds = NewFileMinioClient("config.json.sample", "play") + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "Q3AM3UQ867SPQQA43P2F" { + t.Errorf("Expected 'Q3AM3UQ867SPQQA43P2F', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" { + t.Errorf("Expected 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG', got %s'", credValues.SecretAccessKey) + } + if credValues.SignerType != SignatureV2 { + t.Errorf("Expected 'S3v2', got %s'", credValues.SignerType) + } + + creds = NewFileMinioClient("non-existent.json", "play") + _, err = creds.Get() + if !os.IsNotExist(err) { + t.Errorf("Expected open non-existent.json: no such file or directory, got %s", err) + } + if !creds.IsExpired() { + t.Error("Should be expired if not loaded") + } +} diff --git a/pkg/credentials/iam_aws.go b/pkg/credentials/iam_aws.go new file mode 100644 index 0000000..6845c9a --- /dev/null +++ b/pkg/credentials/iam_aws.go @@ -0,0 +1,252 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "path" + "time" +) + +// DefaultExpiryWindow - Default expiry window. +// ExpiryWindow will allow the credentials to trigger refreshing +// prior to the credentials actually expiring. This is beneficial +// so race conditions with expiring credentials do not cause +// request to fail unexpectedly due to ExpiredTokenException exceptions. +const DefaultExpiryWindow = time.Second * 10 // 10 secs + +// A IAM retrieves credentials from the EC2 service, and keeps track if +// those credentials are expired. +type IAM struct { + Expiry + + // Required http Client to use when connecting to IAM metadata service. + Client *http.Client + + // Custom endpoint to fetch IAM role credentials. + endpoint string +} + +// IAM Roles for Amazon EC2 +// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html +const ( + defaultIAMRoleEndpoint = "http://169.254.169.254" + defaultECSRoleEndpoint = "http://169.254.170.2" + defaultIAMSecurityCredsPath = "/latest/meta-data/iam/security-credentials" +) + +// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html +func getEndpoint(endpoint string) (string, bool) { + if endpoint != "" { + return endpoint, os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") != "" + } + if ecsURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"); ecsURI != "" { + return fmt.Sprintf("%s%s", defaultECSRoleEndpoint, ecsURI), true + } + return defaultIAMRoleEndpoint, false +} + +// NewIAM returns a pointer to a new Credentials object wrapping +// the IAM. Takes a ConfigProvider to create a EC2Metadata client. +// The ConfigProvider is satisfied by the session.Session type. +func NewIAM(endpoint string) *Credentials { + p := &IAM{ + Client: &http.Client{ + Transport: http.DefaultTransport, + }, + endpoint: endpoint, + } + return New(p) +} + +// Retrieve retrieves credentials from the EC2 service. +// Error will be returned if the request fails, or unable to extract +// the desired +func (m *IAM) Retrieve() (Value, error) { + endpoint, isEcsTask := getEndpoint(m.endpoint) + var roleCreds ec2RoleCredRespBody + var err error + if isEcsTask { + roleCreds, err = getEcsTaskCredentials(m.Client, endpoint) + } else { + roleCreds, err = getCredentials(m.Client, endpoint) + } + if err != nil { + return Value{}, err + } + // Expiry window is set to 10secs. + m.SetExpiration(roleCreds.Expiration, DefaultExpiryWindow) + + return Value{ + AccessKeyID: roleCreds.AccessKeyID, + SecretAccessKey: roleCreds.SecretAccessKey, + SessionToken: roleCreds.Token, + SignerType: SignatureV4, + }, nil +} + +// A ec2RoleCredRespBody provides the shape for unmarshaling credential +// request responses. +type ec2RoleCredRespBody struct { + // Success State + Expiration time.Time + AccessKeyID string + SecretAccessKey string + Token string + + // Error state + Code string + Message string + + // Unused params. + LastUpdated time.Time + Type string +} + +// Get the final IAM role URL where the request will +// be sent to fetch the rolling access credentials. +// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html +func getIAMRoleURL(endpoint string) (*url.URL, error) { + u, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + u.Path = defaultIAMSecurityCredsPath + return u, nil +} + +// listRoleNames lists of credential role names associated +// with the current EC2 service. If there are no credentials, +// or there is an error making or receiving the request. +// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html +func listRoleNames(client *http.Client, u *url.URL) ([]string, error) { + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, errors.New(resp.Status) + } + + credsList := []string{} + s := bufio.NewScanner(resp.Body) + for s.Scan() { + credsList = append(credsList, s.Text()) + } + + if err := s.Err(); err != nil { + return nil, err + } + + return credsList, nil +} + +func getEcsTaskCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, error) { + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return ec2RoleCredRespBody{}, err + } + + resp, err := client.Do(req) + if err != nil { + return ec2RoleCredRespBody{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return ec2RoleCredRespBody{}, errors.New(resp.Status) + } + + respCreds := ec2RoleCredRespBody{} + if err := json.NewDecoder(resp.Body).Decode(&respCreds); err != nil { + return ec2RoleCredRespBody{}, err + } + + return respCreds, nil +} + +// getCredentials - obtains the credentials from the IAM role name associated with +// the current EC2 service. +// +// If the credentials cannot be found, or there is an error +// reading the response an error will be returned. +func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, error) { + + // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + u, err := getIAMRoleURL(endpoint) + if err != nil { + return ec2RoleCredRespBody{}, err + } + + // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + roleNames, err := listRoleNames(client, u) + if err != nil { + return ec2RoleCredRespBody{}, err + } + + if len(roleNames) == 0 { + return ec2RoleCredRespBody{}, errors.New("No IAM roles attached to this EC2 service") + } + + // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + // - An instance profile can contain only one IAM role. This limit cannot be increased. + roleName := roleNames[0] + + // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + // The following command retrieves the security credentials for an + // IAM role named `s3access`. + // + // $ curl http://169.254.169.254/latest/meta-data/iam/security-credentials/s3access + // + u.Path = path.Join(u.Path, roleName) + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return ec2RoleCredRespBody{}, err + } + + resp, err := client.Do(req) + if err != nil { + return ec2RoleCredRespBody{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return ec2RoleCredRespBody{}, errors.New(resp.Status) + } + + respCreds := ec2RoleCredRespBody{} + if err := json.NewDecoder(resp.Body).Decode(&respCreds); err != nil { + return ec2RoleCredRespBody{}, err + } + + if respCreds.Code != "Success" { + // If an error code was returned something failed requesting the role. + return ec2RoleCredRespBody{}, errors.New(respCreds.Message) + } + + return respCreds, nil +} diff --git a/pkg/credentials/iam_aws_test.go b/pkg/credentials/iam_aws_test.go new file mode 100644 index 0000000..4dbbb0a --- /dev/null +++ b/pkg/credentials/iam_aws_test.go @@ -0,0 +1,243 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" +) + +const credsRespTmpl = `{ + "Code": "Success", + "Type": "AWS-HMAC", + "AccessKeyId" : "accessKey", + "SecretAccessKey" : "secret", + "Token" : "token", + "Expiration" : "%s", + "LastUpdated" : "2009-11-23T0:00:00Z" +}` + +const credsFailRespTmpl = `{ + "Code": "ErrorCode", + "Message": "ErrorMsg", + "LastUpdated": "2009-11-23T0:00:00Z" +}` + +const credsRespEcsTaskTmpl = `{ + "AccessKeyId" : "accessKey", + "SecretAccessKey" : "secret", + "Token" : "token", + "Expiration" : "%s" +}` + +func initTestFailServer() *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "Not allowed", http.StatusBadRequest) + })) + return server +} + +func initTestServerNoRoles() *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("")) + })) + return server +} + +func initTestServer(expireOn string, failAssume bool) *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/latest/meta-data/iam/security-credentials" { + fmt.Fprintln(w, "RoleName") + } else if r.URL.Path == "/latest/meta-data/iam/security-credentials/RoleName" { + if failAssume { + fmt.Fprintf(w, credsFailRespTmpl) + } else { + fmt.Fprintf(w, credsRespTmpl, expireOn) + } + } else { + http.Error(w, "bad request", http.StatusBadRequest) + } + })) + + return server +} + +func initEcsTaskTestServer(expireOn string) *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, credsRespEcsTaskTmpl, expireOn) + })) + + return server +} + +func TestIAMMalformedEndpoint(t *testing.T) { + creds := NewIAM("%%%%") + _, err := creds.Get() + if err == nil { + t.Fatal("Unexpected should fail here") + } + if err.Error() != `parse %%%%: invalid URL escape "%%%"` { + t.Fatalf("Expected parse %%%%%%%%: invalid URL escape \"%%%%%%\", got %s", err) + } +} + +func TestIAMFailServer(t *testing.T) { + server := initTestFailServer() + defer server.Close() + + creds := NewIAM(server.URL) + + _, err := creds.Get() + if err == nil { + t.Fatal("Unexpected should fail here") + } + if err.Error() != "400 Bad Request" { + t.Fatalf("Expected '400 Bad Request', got %s", err) + } +} + +func TestIAMNoRoles(t *testing.T) { + server := initTestServerNoRoles() + defer server.Close() + + creds := NewIAM(server.URL) + _, err := creds.Get() + if err == nil { + t.Fatal("Unexpected should fail here") + } + if err.Error() != "No IAM roles attached to this EC2 service" { + t.Fatalf("Expected 'No IAM roles attached to this EC2 service', got %s", err) + } +} + +func TestIAM(t *testing.T) { + server := initTestServer("2014-12-16T01:51:37Z", false) + defer server.Close() + + p := &IAM{ + Client: http.DefaultClient, + endpoint: server.URL, + } + + creds, err := p.Retrieve() + if err != nil { + t.Fatal(err) + } + + if "accessKey" != creds.AccessKeyID { + t.Errorf("Expected \"accessKey\", got %s", creds.AccessKeyID) + } + + if "secret" != creds.SecretAccessKey { + t.Errorf("Expected \"secret\", got %s", creds.SecretAccessKey) + } + + if "token" != creds.SessionToken { + t.Errorf("Expected \"token\", got %s", creds.SessionToken) + } + + if !p.IsExpired() { + t.Error("Expected creds to be expired.") + } +} + +func TestIAMFailAssume(t *testing.T) { + server := initTestServer("2014-12-16T01:51:37Z", true) + defer server.Close() + + p := &IAM{ + Client: http.DefaultClient, + endpoint: server.URL, + } + + _, err := p.Retrieve() + if err == nil { + t.Fatal("Unexpected success, should fail") + } + if err.Error() != "ErrorMsg" { + t.Errorf("Expected \"ErrorMsg\", got %s", err) + } +} + +func TestIAMIsExpired(t *testing.T) { + server := initTestServer("2014-12-16T01:51:37Z", false) + defer server.Close() + + p := &IAM{ + Client: http.DefaultClient, + endpoint: server.URL, + } + p.CurrentTime = func() time.Time { + return time.Date(2014, 12, 15, 21, 26, 0, 0, time.UTC) + } + + if !p.IsExpired() { + t.Error("Expected creds to be expired before retrieve.") + } + + _, err := p.Retrieve() + if err != nil { + t.Fatal(err) + } + + if p.IsExpired() { + t.Error("Expected creds to not be expired after retrieve.") + } + + p.CurrentTime = func() time.Time { + return time.Date(3014, 12, 15, 21, 26, 0, 0, time.UTC) + } + + if !p.IsExpired() { + t.Error("Expected creds to be expired when curren time has changed") + } +} + +func TestEcsTask(t *testing.T) { + server := initEcsTaskTestServer("2014-12-16T01:51:37Z") + defer server.Close() + p := &IAM{ + Client: http.DefaultClient, + endpoint: server.URL, + } + os.Setenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "/v2/credentials?id=task_credential_id") + creds, err := p.Retrieve() + os.Unsetenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + if err != nil { + t.Errorf("Unexpected failure %s", err) + } + if "accessKey" != creds.AccessKeyID { + t.Errorf("Expected \"accessKey\", got %s", creds.AccessKeyID) + } + + if "secret" != creds.SecretAccessKey { + t.Errorf("Expected \"secret\", got %s", creds.SecretAccessKey) + } + + if "token" != creds.SessionToken { + t.Errorf("Expected \"token\", got %s", creds.SessionToken) + } + + if !p.IsExpired() { + t.Error("Expected creds to be expired.") + } +} diff --git a/pkg/credentials/signature-type.go b/pkg/credentials/signature-type.go new file mode 100644 index 0000000..1b768e8 --- /dev/null +++ b/pkg/credentials/signature-type.go @@ -0,0 +1,77 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import "strings" + +// SignatureType is type of Authorization requested for a given HTTP request. +type SignatureType int + +// Different types of supported signatures - default is SignatureV4 or SignatureDefault. +const ( + // SignatureDefault is always set to v4. + SignatureDefault SignatureType = iota + SignatureV4 + SignatureV2 + SignatureV4Streaming + SignatureAnonymous // Anonymous signature signifies, no signature. +) + +// IsV2 - is signature SignatureV2? +func (s SignatureType) IsV2() bool { + return s == SignatureV2 +} + +// IsV4 - is signature SignatureV4? +func (s SignatureType) IsV4() bool { + return s == SignatureV4 || s == SignatureDefault +} + +// IsStreamingV4 - is signature SignatureV4Streaming? +func (s SignatureType) IsStreamingV4() bool { + return s == SignatureV4Streaming +} + +// IsAnonymous - is signature empty? +func (s SignatureType) IsAnonymous() bool { + return s == SignatureAnonymous +} + +// Stringer humanized version of signature type, +// strings returned here are case insensitive. +func (s SignatureType) String() string { + if s.IsV2() { + return "S3v2" + } else if s.IsV4() { + return "S3v4" + } else if s.IsStreamingV4() { + return "S3v4Streaming" + } + return "Anonymous" +} + +func parseSignatureType(str string) SignatureType { + if strings.EqualFold(str, "S3v4") { + return SignatureV4 + } else if strings.EqualFold(str, "S3v2") { + return SignatureV2 + } else if strings.EqualFold(str, "S3v4Streaming") { + return SignatureV4Streaming + } + return SignatureAnonymous +} diff --git a/pkg/credentials/static.go b/pkg/credentials/static.go new file mode 100644 index 0000000..8b0ba71 --- /dev/null +++ b/pkg/credentials/static.go @@ -0,0 +1,67 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +// A Static is a set of credentials which are set programmatically, +// and will never expire. +type Static struct { + Value +} + +// NewStaticV2 returns a pointer to a new Credentials object +// wrapping a static credentials value provider, signature is +// set to v2. If access and secret are not specified then +// regardless of signature type set it Value will return +// as anonymous. +func NewStaticV2(id, secret, token string) *Credentials { + return NewStatic(id, secret, token, SignatureV2) +} + +// NewStaticV4 is similar to NewStaticV2 with similar considerations. +func NewStaticV4(id, secret, token string) *Credentials { + return NewStatic(id, secret, token, SignatureV4) +} + +// NewStatic returns a pointer to a new Credentials object +// wrapping a static credentials value provider. +func NewStatic(id, secret, token string, signerType SignatureType) *Credentials { + return New(&Static{ + Value: Value{ + AccessKeyID: id, + SecretAccessKey: secret, + SessionToken: token, + SignerType: signerType, + }, + }) +} + +// Retrieve returns the static credentials. +func (s *Static) Retrieve() (Value, error) { + if s.AccessKeyID == "" || s.SecretAccessKey == "" { + // Anonymous is not an error + return Value{SignerType: SignatureAnonymous}, nil + } + return s.Value, nil +} + +// IsExpired returns if the credentials are expired. +// +// For Static, the credentials never expired. +func (s *Static) IsExpired() bool { + return false +} diff --git a/pkg/credentials/static_test.go b/pkg/credentials/static_test.go new file mode 100644 index 0000000..f1d2d85 --- /dev/null +++ b/pkg/credentials/static_test.go @@ -0,0 +1,68 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import "testing" + +func TestStaticGet(t *testing.T) { + creds := NewStatic("UXHW", "SECRET", "", SignatureV4) + credValues, err := creds.Get() + if err != nil { + t.Fatal(err) + } + + if "UXHW" != credValues.AccessKeyID { + t.Errorf("Expected access key ID to match \"UXHW\", got %s", credValues.AccessKeyID) + } + if "SECRET" != credValues.SecretAccessKey { + t.Errorf("Expected secret access key to match \"SECRET\", got %s", credValues.SecretAccessKey) + } + + if credValues.SessionToken != "" { + t.Error("Expected session token to match") + } + + if credValues.SignerType != SignatureV4 { + t.Errorf("Expected 'S3v4', got %s", credValues.SignerType) + } + + if creds.IsExpired() { + t.Error("Static credentials should never expire") + } + + creds = NewStatic("", "", "", SignatureDefault) + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if "" != credValues.AccessKeyID { + t.Errorf("Expected access key ID to match empty string, got %s", credValues.AccessKeyID) + } + if "" != credValues.SecretAccessKey { + t.Errorf("Expected secret access key to match empty string, got %s", credValues.SecretAccessKey) + } + + if !credValues.SignerType.IsAnonymous() { + t.Errorf("Expected 'Anonymous', got %s", credValues.SignerType) + } + + if creds.IsExpired() { + t.Error("Static credentials should never expire") + } +} diff --git a/pkg/encrypt/server-side.go b/pkg/encrypt/server-side.go new file mode 100644 index 0000000..2d3c70f --- /dev/null +++ b/pkg/encrypt/server-side.go @@ -0,0 +1,195 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package encrypt + +import ( + "crypto/md5" + "encoding/base64" + "encoding/json" + "errors" + "net/http" + + "golang.org/x/crypto/argon2" +) + +const ( + // sseGenericHeader is the AWS SSE header used for SSE-S3 and SSE-KMS. + sseGenericHeader = "X-Amz-Server-Side-Encryption" + + // sseKmsKeyID is the AWS SSE-KMS key id. + sseKmsKeyID = sseGenericHeader + "-Aws-Kms-Key-Id" + // sseEncryptionContext is the AWS SSE-KMS Encryption Context data. + sseEncryptionContext = sseGenericHeader + "-Encryption-Context" + + // sseCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key. + sseCustomerAlgorithm = sseGenericHeader + "-Customer-Algorithm" + // sseCustomerKey is the AWS SSE-C encryption key HTTP header key. + sseCustomerKey = sseGenericHeader + "-Customer-Key" + // sseCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key. + sseCustomerKeyMD5 = sseGenericHeader + "-Customer-Key-MD5" + + // sseCopyCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key for CopyObject API. + sseCopyCustomerAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm" + // sseCopyCustomerKey is the AWS SSE-C encryption key HTTP header key for CopyObject API. + sseCopyCustomerKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key" + // sseCopyCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key for CopyObject API. + sseCopyCustomerKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-MD5" +) + +// PBKDF creates a SSE-C key from the provided password and salt. +// PBKDF is a password-based key derivation function +// which can be used to derive a high-entropy cryptographic +// key from a low-entropy password and a salt. +type PBKDF func(password, salt []byte) ServerSide + +// DefaultPBKDF is the default PBKDF. It uses Argon2id with the +// recommended parameters from the RFC draft (1 pass, 64 MB memory, 4 threads). +var DefaultPBKDF PBKDF = func(password, salt []byte) ServerSide { + sse := ssec{} + copy(sse[:], argon2.IDKey(password, salt, 1, 64*1024, 4, 32)) + return sse +} + +// Type is the server-side-encryption method. It represents one of +// the following encryption methods: +// - SSE-C: server-side-encryption with customer provided keys +// - KMS: server-side-encryption with managed keys +// - S3: server-side-encryption using S3 storage encryption +type Type string + +const ( + // SSEC represents server-side-encryption with customer provided keys + SSEC Type = "SSE-C" + // KMS represents server-side-encryption with managed keys + KMS Type = "KMS" + // S3 represents server-side-encryption using S3 storage encryption + S3 Type = "S3" +) + +// ServerSide is a form of S3 server-side-encryption. +type ServerSide interface { + // Type returns the server-side-encryption method. + Type() Type + + // Marshal adds encryption headers to the provided HTTP headers. + // It marks an HTTP request as server-side-encryption request + // and inserts the required data into the headers. + Marshal(h http.Header) +} + +// NewSSE returns a server-side-encryption using S3 storage encryption. +// Using SSE-S3 the server will encrypt the object with server-managed keys. +func NewSSE() ServerSide { return s3{} } + +// NewSSEKMS returns a new server-side-encryption using SSE-KMS and the provided Key Id and context. +func NewSSEKMS(keyID string, context interface{}) (ServerSide, error) { + if context == nil { + return kms{key: keyID, hasContext: false}, nil + } + serializedContext, err := json.Marshal(context) + if err != nil { + return nil, err + } + return kms{key: keyID, context: serializedContext, hasContext: true}, nil +} + +// NewSSEC returns a new server-side-encryption using SSE-C and the provided key. +// The key must be 32 bytes long. +func NewSSEC(key []byte) (ServerSide, error) { + if len(key) != 32 { + return nil, errors.New("encrypt: SSE-C key must be 256 bit long") + } + sse := ssec{} + copy(sse[:], key) + return sse, nil +} + +// SSE transforms a SSE-C copy encryption into a SSE-C encryption. +// It is the inverse of SSECopy(...). +// +// If the provided sse is no SSE-C copy encryption SSE returns +// sse unmodified. +func SSE(sse ServerSide) ServerSide { + if sse == nil || sse.Type() != SSEC { + return sse + } + if sse, ok := sse.(ssecCopy); ok { + return ssec(sse) + } + return sse +} + +// SSECopy transforms a SSE-C encryption into a SSE-C copy +// encryption. This is required for SSE-C key rotation or a SSE-C +// copy where the source and the destination should be encrypted. +// +// If the provided sse is no SSE-C encryption SSECopy returns +// sse unmodified. +func SSECopy(sse ServerSide) ServerSide { + if sse == nil || sse.Type() != SSEC { + return sse + } + if sse, ok := sse.(ssec); ok { + return ssecCopy(sse) + } + return sse +} + +type ssec [32]byte + +func (s ssec) Type() Type { return SSEC } + +func (s ssec) Marshal(h http.Header) { + keyMD5 := md5.Sum(s[:]) + h.Set(sseCustomerAlgorithm, "AES256") + h.Set(sseCustomerKey, base64.StdEncoding.EncodeToString(s[:])) + h.Set(sseCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:])) +} + +type ssecCopy [32]byte + +func (s ssecCopy) Type() Type { return SSEC } + +func (s ssecCopy) Marshal(h http.Header) { + keyMD5 := md5.Sum(s[:]) + h.Set(sseCopyCustomerAlgorithm, "AES256") + h.Set(sseCopyCustomerKey, base64.StdEncoding.EncodeToString(s[:])) + h.Set(sseCopyCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:])) +} + +type s3 struct{} + +func (s s3) Type() Type { return S3 } + +func (s s3) Marshal(h http.Header) { h.Set(sseGenericHeader, "AES256") } + +type kms struct { + key string + context []byte + hasContext bool +} + +func (s kms) Type() Type { return KMS } + +func (s kms) Marshal(h http.Header) { + h.Set(sseGenericHeader, "aws:kms") + h.Set(sseKmsKeyID, s.key) + if s.hasContext { + h.Set(sseEncryptionContext, base64.StdEncoding.EncodeToString(s.context)) + } +} diff --git a/pkg/policy/bucket-policy-condition.go b/pkg/policy/bucket-policy-condition.go new file mode 100644 index 0000000..737b810 --- /dev/null +++ b/pkg/policy/bucket-policy-condition.go @@ -0,0 +1,116 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package policy + +import "github.com/minio/minio-go/pkg/set" + +// ConditionKeyMap - map of policy condition key and value. +type ConditionKeyMap map[string]set.StringSet + +// Add - adds key and value. The value is appended If key already exists. +func (ckm ConditionKeyMap) Add(key string, value set.StringSet) { + if v, ok := ckm[key]; ok { + ckm[key] = v.Union(value) + } else { + ckm[key] = set.CopyStringSet(value) + } +} + +// Remove - removes value of given key. If key has empty after removal, the key is also removed. +func (ckm ConditionKeyMap) Remove(key string, value set.StringSet) { + if v, ok := ckm[key]; ok { + if value != nil { + ckm[key] = v.Difference(value) + } + + if ckm[key].IsEmpty() { + delete(ckm, key) + } + } +} + +// RemoveKey - removes key and its value. +func (ckm ConditionKeyMap) RemoveKey(key string) { + if _, ok := ckm[key]; ok { + delete(ckm, key) + } +} + +// CopyConditionKeyMap - returns new copy of given ConditionKeyMap. +func CopyConditionKeyMap(condKeyMap ConditionKeyMap) ConditionKeyMap { + out := make(ConditionKeyMap) + + for k, v := range condKeyMap { + out[k] = set.CopyStringSet(v) + } + + return out +} + +// mergeConditionKeyMap - returns a new ConditionKeyMap which contains merged key/value of given two ConditionKeyMap. +func mergeConditionKeyMap(condKeyMap1 ConditionKeyMap, condKeyMap2 ConditionKeyMap) ConditionKeyMap { + out := CopyConditionKeyMap(condKeyMap1) + + for k, v := range condKeyMap2 { + if ev, ok := out[k]; ok { + out[k] = ev.Union(v) + } else { + out[k] = set.CopyStringSet(v) + } + } + + return out +} + +// ConditionMap - map of condition and conditional values. +type ConditionMap map[string]ConditionKeyMap + +// Add - adds condition key and condition value. The value is appended if key already exists. +func (cond ConditionMap) Add(condKey string, condKeyMap ConditionKeyMap) { + if v, ok := cond[condKey]; ok { + cond[condKey] = mergeConditionKeyMap(v, condKeyMap) + } else { + cond[condKey] = CopyConditionKeyMap(condKeyMap) + } +} + +// Remove - removes condition key and its value. +func (cond ConditionMap) Remove(condKey string) { + if _, ok := cond[condKey]; ok { + delete(cond, condKey) + } +} + +// mergeConditionMap - returns new ConditionMap which contains merged key/value of two ConditionMap. +func mergeConditionMap(condMap1 ConditionMap, condMap2 ConditionMap) ConditionMap { + out := make(ConditionMap) + + for k, v := range condMap1 { + out[k] = CopyConditionKeyMap(v) + } + + for k, v := range condMap2 { + if ev, ok := out[k]; ok { + out[k] = mergeConditionKeyMap(ev, v) + } else { + out[k] = CopyConditionKeyMap(v) + } + } + + return out +} diff --git a/pkg/policy/bucket-policy-condition_test.go b/pkg/policy/bucket-policy-condition_test.go new file mode 100644 index 0000000..9e4aa8f --- /dev/null +++ b/pkg/policy/bucket-policy-condition_test.go @@ -0,0 +1,290 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package policy + +import ( + "encoding/json" + "testing" + + "github.com/minio/minio-go/pkg/set" +) + +// ConditionKeyMap.Add() is called and the result is validated. +func TestConditionKeyMapAdd(t *testing.T) { + condKeyMap := make(ConditionKeyMap) + testCases := []struct { + key string + value set.StringSet + expectedResult string + }{ + // Add new key and value. + {"s3:prefix", set.CreateStringSet("hello"), `{"s3:prefix":["hello"]}`}, + // Add existing key and value. + {"s3:prefix", set.CreateStringSet("hello"), `{"s3:prefix":["hello"]}`}, + // Add existing key and not value. + {"s3:prefix", set.CreateStringSet("world"), `{"s3:prefix":["hello","world"]}`}, + } + + for _, testCase := range testCases { + condKeyMap.Add(testCase.key, testCase.value) + if data, err := json.Marshal(condKeyMap); err != nil { + t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) + } else { + if string(data) != testCase.expectedResult { + t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) + } + } + } +} + +// ConditionKeyMap.Remove() is called and the result is validated. +func TestConditionKeyMapRemove(t *testing.T) { + condKeyMap := make(ConditionKeyMap) + condKeyMap.Add("s3:prefix", set.CreateStringSet("hello", "world")) + + testCases := []struct { + key string + value set.StringSet + expectedResult string + }{ + // Remove non-existent key and value. + {"s3:myprefix", set.CreateStringSet("hello"), `{"s3:prefix":["hello","world"]}`}, + // Remove existing key and value. + {"s3:prefix", set.CreateStringSet("hello"), `{"s3:prefix":["world"]}`}, + // Remove existing key to make the key also removed. + {"s3:prefix", set.CreateStringSet("world"), `{}`}, + } + + for _, testCase := range testCases { + condKeyMap.Remove(testCase.key, testCase.value) + if data, err := json.Marshal(condKeyMap); err != nil { + t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) + } else { + if string(data) != testCase.expectedResult { + t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) + } + } + } +} + +// ConditionKeyMap.RemoveKey() is called and the result is validated. +func TestConditionKeyMapRemoveKey(t *testing.T) { + condKeyMap := make(ConditionKeyMap) + condKeyMap.Add("s3:prefix", set.CreateStringSet("hello", "world")) + + testCases := []struct { + key string + expectedResult string + }{ + // Remove non-existent key. + {"s3:myprefix", `{"s3:prefix":["hello","world"]}`}, + // Remove existing key. + {"s3:prefix", `{}`}, + } + + for _, testCase := range testCases { + condKeyMap.RemoveKey(testCase.key) + if data, err := json.Marshal(condKeyMap); err != nil { + t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) + } else { + if string(data) != testCase.expectedResult { + t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) + } + } + } +} + +// CopyConditionKeyMap() is called and the result is validated. +func TestCopyConditionKeyMap(t *testing.T) { + emptyCondKeyMap := make(ConditionKeyMap) + nonEmptyCondKeyMap := make(ConditionKeyMap) + nonEmptyCondKeyMap.Add("s3:prefix", set.CreateStringSet("hello", "world")) + + testCases := []struct { + condKeyMap ConditionKeyMap + expectedResult string + }{ + // To test empty ConditionKeyMap. + {emptyCondKeyMap, `{}`}, + // To test non-empty ConditionKeyMap. + {nonEmptyCondKeyMap, `{"s3:prefix":["hello","world"]}`}, + } + + for _, testCase := range testCases { + condKeyMap := CopyConditionKeyMap(testCase.condKeyMap) + if data, err := json.Marshal(condKeyMap); err != nil { + t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) + } else { + if string(data) != testCase.expectedResult { + t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) + } + } + } +} + +// mergeConditionKeyMap() is called and the result is validated. +func TestMergeConditionKeyMap(t *testing.T) { + condKeyMap1 := make(ConditionKeyMap) + condKeyMap1.Add("s3:prefix", set.CreateStringSet("hello")) + + condKeyMap2 := make(ConditionKeyMap) + condKeyMap2.Add("s3:prefix", set.CreateStringSet("world")) + + condKeyMap3 := make(ConditionKeyMap) + condKeyMap3.Add("s3:myprefix", set.CreateStringSet("world")) + + testCases := []struct { + condKeyMap1 ConditionKeyMap + condKeyMap2 ConditionKeyMap + expectedResult string + }{ + // Both arguments are empty. + {make(ConditionKeyMap), make(ConditionKeyMap), `{}`}, + // First argument is empty. + {make(ConditionKeyMap), condKeyMap1, `{"s3:prefix":["hello"]}`}, + // Second argument is empty. + {condKeyMap1, make(ConditionKeyMap), `{"s3:prefix":["hello"]}`}, + // Both arguments are same value. + {condKeyMap1, condKeyMap1, `{"s3:prefix":["hello"]}`}, + // Value of second argument will be merged. + {condKeyMap1, condKeyMap2, `{"s3:prefix":["hello","world"]}`}, + // second argument will be added. + {condKeyMap1, condKeyMap3, `{"s3:myprefix":["world"],"s3:prefix":["hello"]}`}, + } + + for _, testCase := range testCases { + condKeyMap := mergeConditionKeyMap(testCase.condKeyMap1, testCase.condKeyMap2) + if data, err := json.Marshal(condKeyMap); err != nil { + t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) + } else { + if string(data) != testCase.expectedResult { + t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) + } + } + } +} + +// ConditionMap.Add() is called and the result is validated. +func TestConditionMapAdd(t *testing.T) { + condMap := make(ConditionMap) + + condKeyMap1 := make(ConditionKeyMap) + condKeyMap1.Add("s3:prefix", set.CreateStringSet("hello")) + + condKeyMap2 := make(ConditionKeyMap) + condKeyMap2.Add("s3:prefix", set.CreateStringSet("hello", "world")) + + testCases := []struct { + key string + value ConditionKeyMap + expectedResult string + }{ + // Add new key and value. + {"StringEquals", condKeyMap1, `{"StringEquals":{"s3:prefix":["hello"]}}`}, + // Add existing key and value. + {"StringEquals", condKeyMap1, `{"StringEquals":{"s3:prefix":["hello"]}}`}, + // Add existing key and not value. + {"StringEquals", condKeyMap2, `{"StringEquals":{"s3:prefix":["hello","world"]}}`}, + } + + for _, testCase := range testCases { + condMap.Add(testCase.key, testCase.value) + if data, err := json.Marshal(condMap); err != nil { + t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) + } else { + if string(data) != testCase.expectedResult { + t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) + } + } + } +} + +// ConditionMap.Remove() is called and the result is validated. +func TestConditionMapRemove(t *testing.T) { + condMap := make(ConditionMap) + condKeyMap := make(ConditionKeyMap) + condKeyMap.Add("s3:prefix", set.CreateStringSet("hello", "world")) + condMap.Add("StringEquals", condKeyMap) + + testCases := []struct { + key string + expectedResult string + }{ + // Remove non-existent key. + {"StringNotEquals", `{"StringEquals":{"s3:prefix":["hello","world"]}}`}, + // Remove existing key. + {"StringEquals", `{}`}, + } + + for _, testCase := range testCases { + condMap.Remove(testCase.key) + if data, err := json.Marshal(condMap); err != nil { + t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) + } else { + if string(data) != testCase.expectedResult { + t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) + } + } + } +} + +// mergeConditionMap() is called and the result is validated. +func TestMergeConditionMap(t *testing.T) { + condKeyMap1 := make(ConditionKeyMap) + condKeyMap1.Add("s3:prefix", set.CreateStringSet("hello")) + condMap1 := make(ConditionMap) + condMap1.Add("StringEquals", condKeyMap1) + + condKeyMap2 := make(ConditionKeyMap) + condKeyMap2.Add("s3:prefix", set.CreateStringSet("world")) + condMap2 := make(ConditionMap) + condMap2.Add("StringEquals", condKeyMap2) + + condMap3 := make(ConditionMap) + condMap3.Add("StringNotEquals", condKeyMap2) + + testCases := []struct { + condMap1 ConditionMap + condMap2 ConditionMap + expectedResult string + }{ + // Both arguments are empty. + {make(ConditionMap), make(ConditionMap), `{}`}, + // First argument is empty. + {make(ConditionMap), condMap1, `{"StringEquals":{"s3:prefix":["hello"]}}`}, + // Second argument is empty. + {condMap1, make(ConditionMap), `{"StringEquals":{"s3:prefix":["hello"]}}`}, + // Both arguments are same value. + {condMap1, condMap1, `{"StringEquals":{"s3:prefix":["hello"]}}`}, + // Value of second argument will be merged. + {condMap1, condMap2, `{"StringEquals":{"s3:prefix":["hello","world"]}}`}, + // second argument will be added. + {condMap1, condMap3, `{"StringEquals":{"s3:prefix":["hello"]},"StringNotEquals":{"s3:prefix":["world"]}}`}, + } + + for _, testCase := range testCases { + condMap := mergeConditionMap(testCase.condMap1, testCase.condMap2) + if data, err := json.Marshal(condMap); err != nil { + t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) + } else { + if string(data) != testCase.expectedResult { + t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) + } + } + } +} diff --git a/pkg/policy/bucket-policy.go b/pkg/policy/bucket-policy.go new file mode 100644 index 0000000..79fd801 --- /dev/null +++ b/pkg/policy/bucket-policy.go @@ -0,0 +1,668 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package policy + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + + "github.com/minio/minio-go/pkg/set" +) + +// BucketPolicy - Bucket level policy. +type BucketPolicy string + +// Different types of Policies currently supported for buckets. +const ( + BucketPolicyNone BucketPolicy = "none" + BucketPolicyReadOnly = "readonly" + BucketPolicyReadWrite = "readwrite" + BucketPolicyWriteOnly = "writeonly" +) + +// IsValidBucketPolicy - returns true if policy is valid and supported, false otherwise. +func (p BucketPolicy) IsValidBucketPolicy() bool { + switch p { + case BucketPolicyNone, BucketPolicyReadOnly, BucketPolicyReadWrite, BucketPolicyWriteOnly: + return true + } + return false +} + +// Resource prefix for all aws resources. +const awsResourcePrefix = "arn:aws:s3:::" + +// Common bucket actions for both read and write policies. +var commonBucketActions = set.CreateStringSet("s3:GetBucketLocation") + +// Read only bucket actions. +var readOnlyBucketActions = set.CreateStringSet("s3:ListBucket") + +// Write only bucket actions. +var writeOnlyBucketActions = set.CreateStringSet("s3:ListBucketMultipartUploads") + +// Read only object actions. +var readOnlyObjectActions = set.CreateStringSet("s3:GetObject") + +// Write only object actions. +var writeOnlyObjectActions = set.CreateStringSet("s3:AbortMultipartUpload", "s3:DeleteObject", "s3:ListMultipartUploadParts", "s3:PutObject") + +// Read and write object actions. +var readWriteObjectActions = readOnlyObjectActions.Union(writeOnlyObjectActions) + +// All valid bucket and object actions. +var validActions = commonBucketActions. + Union(readOnlyBucketActions). + Union(writeOnlyBucketActions). + Union(readOnlyObjectActions). + Union(writeOnlyObjectActions) + +var startsWithFunc = func(resource string, resourcePrefix string) bool { + return strings.HasPrefix(resource, resourcePrefix) +} + +// User - canonical users list. +type User struct { + AWS set.StringSet `json:"AWS,omitempty"` + CanonicalUser set.StringSet `json:"CanonicalUser,omitempty"` +} + +// UnmarshalJSON is a custom json unmarshaler for Principal field, +// the reason is that Principal can take a json struct represented by +// User string but it can also take a string. +func (u *User) UnmarshalJSON(data []byte) error { + // Try to unmarshal data in a struct equal to User, we need it + // to avoid infinite recursive call of this function + type AliasUser User + var au AliasUser + err := json.Unmarshal(data, &au) + if err == nil { + *u = User(au) + return nil + } + // Data type is not known, check if it is a json string + // which contains a star, which is permitted in the spec + var str string + err = json.Unmarshal(data, &str) + if err == nil { + if str != "*" { + return errors.New("unrecognized Principal field") + } + *u = User{AWS: set.CreateStringSet("*")} + return nil + } + return err +} + +// Statement - minio policy statement +type Statement struct { + Actions set.StringSet `json:"Action"` + Conditions ConditionMap `json:"Condition,omitempty"` + Effect string + Principal User `json:"Principal"` + Resources set.StringSet `json:"Resource"` + Sid string +} + +// BucketAccessPolicy - minio policy collection +type BucketAccessPolicy struct { + Version string // date in YYYY-MM-DD format + Statements []Statement `json:"Statement"` +} + +// isValidStatement - returns whether given statement is valid to process for given bucket name. +func isValidStatement(statement Statement, bucketName string) bool { + if statement.Actions.Intersection(validActions).IsEmpty() { + return false + } + + if statement.Effect != "Allow" { + return false + } + + if statement.Principal.AWS == nil || !statement.Principal.AWS.Contains("*") { + return false + } + + bucketResource := awsResourcePrefix + bucketName + if statement.Resources.Contains(bucketResource) { + return true + } + + if statement.Resources.FuncMatch(startsWithFunc, bucketResource+"/").IsEmpty() { + return false + } + + return true +} + +// Returns new statements with bucket actions for given policy. +func newBucketStatement(policy BucketPolicy, bucketName string, prefix string) (statements []Statement) { + statements = []Statement{} + if policy == BucketPolicyNone || bucketName == "" { + return statements + } + + bucketResource := set.CreateStringSet(awsResourcePrefix + bucketName) + + statement := Statement{ + Actions: commonBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: bucketResource, + Sid: "", + } + statements = append(statements, statement) + + if policy == BucketPolicyReadOnly || policy == BucketPolicyReadWrite { + statement = Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: bucketResource, + Sid: "", + } + if prefix != "" { + condKeyMap := make(ConditionKeyMap) + condKeyMap.Add("s3:prefix", set.CreateStringSet(prefix)) + condMap := make(ConditionMap) + condMap.Add("StringEquals", condKeyMap) + statement.Conditions = condMap + } + statements = append(statements, statement) + } + + if policy == BucketPolicyWriteOnly || policy == BucketPolicyReadWrite { + statement = Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: bucketResource, + Sid: "", + } + statements = append(statements, statement) + } + + return statements +} + +// Returns new statements contains object actions for given policy. +func newObjectStatement(policy BucketPolicy, bucketName string, prefix string) (statements []Statement) { + statements = []Statement{} + if policy == BucketPolicyNone || bucketName == "" { + return statements + } + + statement := Statement{ + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet(awsResourcePrefix + bucketName + "/" + prefix + "*"), + Sid: "", + } + + if policy == BucketPolicyReadOnly { + statement.Actions = readOnlyObjectActions + } else if policy == BucketPolicyWriteOnly { + statement.Actions = writeOnlyObjectActions + } else if policy == BucketPolicyReadWrite { + statement.Actions = readWriteObjectActions + } + + statements = append(statements, statement) + return statements +} + +// Returns new statements for given policy, bucket and prefix. +func newStatements(policy BucketPolicy, bucketName string, prefix string) (statements []Statement) { + statements = []Statement{} + ns := newBucketStatement(policy, bucketName, prefix) + statements = append(statements, ns...) + + ns = newObjectStatement(policy, bucketName, prefix) + statements = append(statements, ns...) + + return statements +} + +// Returns whether given bucket statements are used by other than given prefix statements. +func getInUsePolicy(statements []Statement, bucketName string, prefix string) (readOnlyInUse, writeOnlyInUse bool) { + resourcePrefix := awsResourcePrefix + bucketName + "/" + objectResource := awsResourcePrefix + bucketName + "/" + prefix + "*" + + for _, s := range statements { + if !s.Resources.Contains(objectResource) && !s.Resources.FuncMatch(startsWithFunc, resourcePrefix).IsEmpty() { + if s.Actions.Intersection(readOnlyObjectActions).Equals(readOnlyObjectActions) { + readOnlyInUse = true + } + + if s.Actions.Intersection(writeOnlyObjectActions).Equals(writeOnlyObjectActions) { + writeOnlyInUse = true + } + } + if readOnlyInUse && writeOnlyInUse { + break + } + } + + return readOnlyInUse, writeOnlyInUse +} + +// Removes object actions in given statement. +func removeObjectActions(statement Statement, objectResource string) Statement { + if statement.Conditions == nil { + if len(statement.Resources) > 1 { + statement.Resources.Remove(objectResource) + } else { + statement.Actions = statement.Actions.Difference(readOnlyObjectActions) + statement.Actions = statement.Actions.Difference(writeOnlyObjectActions) + } + } + + return statement +} + +// Removes bucket actions for given policy in given statement. +func removeBucketActions(statement Statement, prefix string, bucketResource string, readOnlyInUse, writeOnlyInUse bool) Statement { + removeReadOnly := func() { + if !statement.Actions.Intersection(readOnlyBucketActions).Equals(readOnlyBucketActions) { + return + } + + if statement.Conditions == nil { + statement.Actions = statement.Actions.Difference(readOnlyBucketActions) + return + } + + if prefix != "" { + stringEqualsValue := statement.Conditions["StringEquals"] + values := set.NewStringSet() + if stringEqualsValue != nil { + values = stringEqualsValue["s3:prefix"] + if values == nil { + values = set.NewStringSet() + } + } + + values.Remove(prefix) + + if stringEqualsValue != nil { + if values.IsEmpty() { + delete(stringEqualsValue, "s3:prefix") + } + if len(stringEqualsValue) == 0 { + delete(statement.Conditions, "StringEquals") + } + } + + if len(statement.Conditions) == 0 { + statement.Conditions = nil + statement.Actions = statement.Actions.Difference(readOnlyBucketActions) + } + } + } + + removeWriteOnly := func() { + if statement.Conditions == nil { + statement.Actions = statement.Actions.Difference(writeOnlyBucketActions) + } + } + + if len(statement.Resources) > 1 { + statement.Resources.Remove(bucketResource) + } else { + if !readOnlyInUse { + removeReadOnly() + } + + if !writeOnlyInUse { + removeWriteOnly() + } + } + + return statement +} + +// Returns statements containing removed actions/statements for given +// policy, bucket name and prefix. +func removeStatements(statements []Statement, bucketName string, prefix string) []Statement { + bucketResource := awsResourcePrefix + bucketName + objectResource := awsResourcePrefix + bucketName + "/" + prefix + "*" + readOnlyInUse, writeOnlyInUse := getInUsePolicy(statements, bucketName, prefix) + + out := []Statement{} + readOnlyBucketStatements := []Statement{} + s3PrefixValues := set.NewStringSet() + + for _, statement := range statements { + if !isValidStatement(statement, bucketName) { + out = append(out, statement) + continue + } + + if statement.Resources.Contains(bucketResource) { + if statement.Conditions != nil { + statement = removeBucketActions(statement, prefix, bucketResource, false, false) + } else { + statement = removeBucketActions(statement, prefix, bucketResource, readOnlyInUse, writeOnlyInUse) + } + } else if statement.Resources.Contains(objectResource) { + statement = removeObjectActions(statement, objectResource) + } + + if !statement.Actions.IsEmpty() { + if statement.Resources.Contains(bucketResource) && + statement.Actions.Intersection(readOnlyBucketActions).Equals(readOnlyBucketActions) && + statement.Effect == "Allow" && + statement.Principal.AWS.Contains("*") { + + if statement.Conditions != nil { + stringEqualsValue := statement.Conditions["StringEquals"] + values := set.NewStringSet() + if stringEqualsValue != nil { + values = stringEqualsValue["s3:prefix"] + if values == nil { + values = set.NewStringSet() + } + } + s3PrefixValues = s3PrefixValues.Union(values.ApplyFunc(func(v string) string { + return bucketResource + "/" + v + "*" + })) + } else if !s3PrefixValues.IsEmpty() { + readOnlyBucketStatements = append(readOnlyBucketStatements, statement) + continue + } + } + out = append(out, statement) + } + } + + skipBucketStatement := true + resourcePrefix := awsResourcePrefix + bucketName + "/" + for _, statement := range out { + if !statement.Resources.FuncMatch(startsWithFunc, resourcePrefix).IsEmpty() && + s3PrefixValues.Intersection(statement.Resources).IsEmpty() { + skipBucketStatement = false + break + } + } + + for _, statement := range readOnlyBucketStatements { + if skipBucketStatement && + statement.Resources.Contains(bucketResource) && + statement.Effect == "Allow" && + statement.Principal.AWS.Contains("*") && + statement.Conditions == nil { + continue + } + + out = append(out, statement) + } + + if len(out) == 1 { + statement := out[0] + if statement.Resources.Contains(bucketResource) && + statement.Actions.Intersection(commonBucketActions).Equals(commonBucketActions) && + statement.Effect == "Allow" && + statement.Principal.AWS.Contains("*") && + statement.Conditions == nil { + out = []Statement{} + } + } + + return out +} + +// Appends given statement into statement list to have unique statements. +// - If statement already exists in statement list, it ignores. +// - If statement exists with different conditions, they are merged. +// - Else the statement is appended to statement list. +func appendStatement(statements []Statement, statement Statement) []Statement { + for i, s := range statements { + if s.Actions.Equals(statement.Actions) && + s.Effect == statement.Effect && + s.Principal.AWS.Equals(statement.Principal.AWS) && + reflect.DeepEqual(s.Conditions, statement.Conditions) { + statements[i].Resources = s.Resources.Union(statement.Resources) + return statements + } else if s.Resources.Equals(statement.Resources) && + s.Effect == statement.Effect && + s.Principal.AWS.Equals(statement.Principal.AWS) && + reflect.DeepEqual(s.Conditions, statement.Conditions) { + statements[i].Actions = s.Actions.Union(statement.Actions) + return statements + } + + if s.Resources.Intersection(statement.Resources).Equals(statement.Resources) && + s.Actions.Intersection(statement.Actions).Equals(statement.Actions) && + s.Effect == statement.Effect && + s.Principal.AWS.Intersection(statement.Principal.AWS).Equals(statement.Principal.AWS) { + if reflect.DeepEqual(s.Conditions, statement.Conditions) { + return statements + } + if s.Conditions != nil && statement.Conditions != nil { + if s.Resources.Equals(statement.Resources) { + statements[i].Conditions = mergeConditionMap(s.Conditions, statement.Conditions) + return statements + } + } + } + } + + if !(statement.Actions.IsEmpty() && statement.Resources.IsEmpty()) { + return append(statements, statement) + } + + return statements +} + +// Appends two statement lists. +func appendStatements(statements []Statement, appendStatements []Statement) []Statement { + for _, s := range appendStatements { + statements = appendStatement(statements, s) + } + + return statements +} + +// Returns policy of given bucket statement. +func getBucketPolicy(statement Statement, prefix string) (commonFound, readOnly, writeOnly bool) { + if !(statement.Effect == "Allow" && statement.Principal.AWS.Contains("*")) { + return commonFound, readOnly, writeOnly + } + + if statement.Actions.Intersection(commonBucketActions).Equals(commonBucketActions) && + statement.Conditions == nil { + commonFound = true + } + + if statement.Actions.Intersection(writeOnlyBucketActions).Equals(writeOnlyBucketActions) && + statement.Conditions == nil { + writeOnly = true + } + + if statement.Actions.Intersection(readOnlyBucketActions).Equals(readOnlyBucketActions) { + if prefix != "" && statement.Conditions != nil { + if stringEqualsValue, ok := statement.Conditions["StringEquals"]; ok { + if s3PrefixValues, ok := stringEqualsValue["s3:prefix"]; ok { + if s3PrefixValues.Contains(prefix) { + readOnly = true + } + } + } else if stringNotEqualsValue, ok := statement.Conditions["StringNotEquals"]; ok { + if s3PrefixValues, ok := stringNotEqualsValue["s3:prefix"]; ok { + if !s3PrefixValues.Contains(prefix) { + readOnly = true + } + } + } + } else if prefix == "" && statement.Conditions == nil { + readOnly = true + } else if prefix != "" && statement.Conditions == nil { + readOnly = true + } + } + + return commonFound, readOnly, writeOnly +} + +// Returns policy of given object statement. +func getObjectPolicy(statement Statement) (readOnly bool, writeOnly bool) { + if statement.Effect == "Allow" && + statement.Principal.AWS.Contains("*") && + statement.Conditions == nil { + if statement.Actions.Intersection(readOnlyObjectActions).Equals(readOnlyObjectActions) { + readOnly = true + } + if statement.Actions.Intersection(writeOnlyObjectActions).Equals(writeOnlyObjectActions) { + writeOnly = true + } + } + + return readOnly, writeOnly +} + +// GetPolicy - Returns policy of given bucket name, prefix in given statements. +func GetPolicy(statements []Statement, bucketName string, prefix string) BucketPolicy { + bucketResource := awsResourcePrefix + bucketName + objectResource := awsResourcePrefix + bucketName + "/" + prefix + "*" + + bucketCommonFound := false + bucketReadOnly := false + bucketWriteOnly := false + matchedResource := "" + objReadOnly := false + objWriteOnly := false + + for _, s := range statements { + matchedObjResources := set.NewStringSet() + if s.Resources.Contains(objectResource) { + matchedObjResources.Add(objectResource) + } else { + matchedObjResources = s.Resources.FuncMatch(resourceMatch, objectResource) + } + if !matchedObjResources.IsEmpty() { + readOnly, writeOnly := getObjectPolicy(s) + for resource := range matchedObjResources { + if len(matchedResource) < len(resource) { + objReadOnly = readOnly + objWriteOnly = writeOnly + matchedResource = resource + } else if len(matchedResource) == len(resource) { + objReadOnly = objReadOnly || readOnly + objWriteOnly = objWriteOnly || writeOnly + matchedResource = resource + } + } + } + if s.Resources.Contains(bucketResource) { + commonFound, readOnly, writeOnly := getBucketPolicy(s, prefix) + bucketCommonFound = bucketCommonFound || commonFound + bucketReadOnly = bucketReadOnly || readOnly + bucketWriteOnly = bucketWriteOnly || writeOnly + } + } + + policy := BucketPolicyNone + if bucketCommonFound { + if bucketReadOnly && bucketWriteOnly && objReadOnly && objWriteOnly { + policy = BucketPolicyReadWrite + } else if bucketReadOnly && objReadOnly { + policy = BucketPolicyReadOnly + } else if bucketWriteOnly && objWriteOnly { + policy = BucketPolicyWriteOnly + } + } + + return policy +} + +// GetPolicies - returns a map of policies of given bucket name, prefix in given statements. +func GetPolicies(statements []Statement, bucketName, prefix string) map[string]BucketPolicy { + policyRules := map[string]BucketPolicy{} + objResources := set.NewStringSet() + // Search all resources related to objects policy + for _, s := range statements { + for r := range s.Resources { + if strings.HasPrefix(r, awsResourcePrefix+bucketName+"/"+prefix) { + objResources.Add(r) + } + } + } + + // Pretend that policy resource as an actual object and fetch its policy + for r := range objResources { + // Put trailing * if exists in asterisk + asterisk := "" + if strings.HasSuffix(r, "*") { + r = r[:len(r)-1] + asterisk = "*" + } + var objectPath string + if len(r) >= len(awsResourcePrefix+bucketName)+1 { + objectPath = r[len(awsResourcePrefix+bucketName)+1:] + } + p := GetPolicy(statements, bucketName, objectPath) + policyRules[bucketName+"/"+objectPath+asterisk] = p + } + return policyRules +} + +// SetPolicy - Returns new statements containing policy of given bucket name and prefix are appended. +func SetPolicy(statements []Statement, policy BucketPolicy, bucketName string, prefix string) []Statement { + out := removeStatements(statements, bucketName, prefix) + // fmt.Println("out = ") + // printstatement(out) + ns := newStatements(policy, bucketName, prefix) + // fmt.Println("ns = ") + // printstatement(ns) + + rv := appendStatements(out, ns) + // fmt.Println("rv = ") + // printstatement(rv) + + return rv +} + +// Match function matches wild cards in 'pattern' for resource. +func resourceMatch(pattern, resource string) bool { + if pattern == "" { + return resource == pattern + } + if pattern == "*" { + return true + } + parts := strings.Split(pattern, "*") + if len(parts) == 1 { + return resource == pattern + } + tGlob := strings.HasSuffix(pattern, "*") + end := len(parts) - 1 + if !strings.HasPrefix(resource, parts[0]) { + return false + } + for i := 1; i < end; i++ { + if !strings.Contains(resource, parts[i]) { + return false + } + idx := strings.Index(resource, parts[i]) + len(parts[i]) + resource = resource[idx:] + } + return tGlob || strings.HasSuffix(resource, parts[end]) +} diff --git a/pkg/policy/bucket-policy_test.go b/pkg/policy/bucket-policy_test.go new file mode 100644 index 0000000..1a71d87 --- /dev/null +++ b/pkg/policy/bucket-policy_test.go @@ -0,0 +1,2030 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package policy + +import ( + "encoding/json" + "fmt" + "reflect" + "testing" + + "github.com/minio/minio-go/pkg/set" +) + +// TestUnmarshalBucketPolicy tests unmarsheling various examples +// of bucket policies, to verify the correctness of BucketAccessPolicy +// struct defined in this package. +func TestUnmarshalBucketPolicy(t *testing.T) { + var testCases = []struct { + policyData string + shouldSucceed bool + }{ + // Test 1 + {policyData: `{ + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"AddCannedAcl", + "Effect":"Allow", + "Principal": {"AWS": ["arn:aws:iam::111122223333:root","arn:aws:iam::444455556666:root"]}, + "Action":["s3:PutObject","s3:PutObjectAcl"], + "Resource":["arn:aws:s3:::examplebucket/*"], + "Condition":{"StringEquals":{"s3:x-amz-acl":["public-read"]}} + } + ] +}`, shouldSucceed: true}, + // Test 2 + {policyData: `{ + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"AddPerm", + "Effect":"Allow", + "Principal": "*", + "Action":["s3:GetObject"], + "Resource":["arn:aws:s3:::examplebucket/*"] + } + ] +}`, shouldSucceed: true}, + // Test 3 + {policyData: `{ + "Version": "2012-10-17", + "Id": "S3PolicyId1", + "Statement": [ + { + "Sid": "IPAllow", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:*", + "Resource": "arn:aws:s3:::examplebucket/*", + "Condition": { + "IpAddress": {"aws:SourceIp": "54.240.143.0/24"}, + "NotIpAddress": {"aws:SourceIp": "54.240.143.188/32"} + } + } + ] +}`, shouldSucceed: true}, + // Test 4 + {policyData: `{ + "Id":"PolicyId2", + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"AllowIPmix", + "Effect":"Allow", + "Principal":"*", + "Action":"s3:*", + "Resource":"arn:aws:s3:::examplebucket/*", + "Condition": { + "IpAddress": { + "aws:SourceIp": [ + "54.240.143.0/24", + "2001:DB8:1234:5678::/64" + ] + }, + "NotIpAddress": { + "aws:SourceIp": [ + "54.240.143.128/30", + "2001:DB8:1234:5678:ABCD::/80" + ] + } + } + } + ] +}`, shouldSucceed: true}, + // Test 5 + {policyData: `{ + "Version":"2012-10-17", + "Id":"http referer policy example", + "Statement":[ + { + "Sid":"Allow get requests originating from www.example.com and example.com.", + "Effect":"Allow", + "Principal":"*", + "Action":"s3:GetObject", + "Resource":"arn:aws:s3:::examplebucket/*", + "Condition":{ + "StringLike":{"aws:Referer":["http://www.example.com/*","http://example.com/*"]} + } + } + ] +}`, shouldSucceed: true}, + // Test 6 + {policyData: `{ + "Version": "2012-10-17", + "Id": "http referer policy example", + "Statement": [ + { + "Sid": "Allow get requests referred by www.example.com and example.com.", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::examplebucket/*", + "Condition": { + "StringLike": {"aws:Referer": ["http://www.example.com/*","http://example.com/*"]} + } + }, + { + "Sid": "Explicit deny to ensure requests are allowed only from specific referer.", + "Effect": "Deny", + "Principal": "*", + "Action": "s3:*", + "Resource": "arn:aws:s3:::examplebucket/*", + "Condition": { + "StringNotLike": {"aws:Referer": ["http://www.example.com/*","http://example.com/*"]} + } + } + ] +}`, shouldSucceed: true}, + + // Test 7 + {policyData: `{ + "Version":"2012-10-17", + "Id":"PolicyForCloudFrontPrivateContent", + "Statement":[ + { + "Sid":" Grant a CloudFront Origin Identity access to support private content", + "Effect":"Allow", + "Principal":{"CanonicalUser":"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be"}, + "Action":"s3:GetObject", + "Resource":"arn:aws:s3:::example-bucket/*" + } + ] +}`, shouldSucceed: true}, + // Test 8 + {policyData: `{ + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"111", + "Effect":"Allow", + "Principal":{"AWS":"1111111111"}, + "Action":"s3:PutObject", + "Resource":"arn:aws:s3:::examplebucket/*" + }, + { + "Sid":"112", + "Effect":"Deny", + "Principal":{"AWS":"1111111111" }, + "Action":"s3:PutObject", + "Resource":"arn:aws:s3:::examplebucket/*", + "Condition": { + "StringNotEquals": {"s3:x-amz-grant-full-control":["emailAddress=xyz@amazon.com"]} + } + } + ] +}`, shouldSucceed: true}, + // Test 9 + {policyData: `{ + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"InventoryAndAnalyticsExamplePolicy", + "Effect":"Allow", + "Principal": {"Service": "s3.amazonaws.com"}, + "Action":["s3:PutObject"], + "Resource":["arn:aws:s3:::destination-bucket/*"], + "Condition": { + "ArnLike": { + "aws:SourceArn": "arn:aws:s3:::source-bucket" + }, + "StringEquals": { + "aws:SourceAccount": "1234567890", + "s3:x-amz-acl": "bucket-owner-full-control" + } + } + } + ] +}`, shouldSucceed: true}, + } + + for i, testCase := range testCases { + var policy BucketAccessPolicy + err := json.Unmarshal([]byte(testCase.policyData), &policy) + if testCase.shouldSucceed && err != nil { + t.Fatalf("Test %d: expected to succeed but it has an error: %v", i+1, err) + } + if !testCase.shouldSucceed && err == nil { + t.Fatalf("Test %d: expected to fail but succeeded", i+1) + } + } +} + +// isValidStatement() is called and the result is validated. +func TestIsValidStatement(t *testing.T) { + testCases := []struct { + statement Statement + bucketName string + expectedResult bool + }{ + // Empty statement and bucket name. + {Statement{}, "", false}, + // Empty statement. + {Statement{}, "mybucket", false}, + // Empty bucket name. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "", false}, + // Statement with unknown actions. + {Statement{ + Actions: set.CreateStringSet("s3:ListBucketVersions"), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "mybucket", false}, + // Statement with unknown effect. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Deny", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "mybucket", false}, + // Statement with nil Principal.AWS. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "mybucket", false}, + // Statement with unknown Principal.AWS. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("arn:aws:iam::AccountNumberWithoutHyphens:root")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "mybucket", false}, + // Statement with different bucket name. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), + }, "mybucket", false}, + // Statement with bucket name with suffixed string. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybuckettest/myobject"), + }, "mybucket", false}, + // Statement with bucket name and object name. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/myobject"), + }, "mybucket", true}, + // Statement with condition, bucket name and object name. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/myobject"), + }, "mybucket", true}, + } + + for _, testCase := range testCases { + if result := isValidStatement(testCase.statement, testCase.bucketName); result != testCase.expectedResult { + t.Fatalf("%+v: expected: %t, got: %t", testCase, testCase.expectedResult, result) + } + } +} + +// newStatements() is called and the result is validated. +func TestNewStatements(t *testing.T) { + testCases := []struct { + policy BucketPolicy + bucketName string + prefix string + expectedResult string + }{ + // BucketPolicyNone: with empty bucket name and prefix. + {BucketPolicyNone, "", "", `[]`}, + // BucketPolicyNone: with bucket name and empty prefix. + {BucketPolicyNone, "mybucket", "", `[]`}, + // BucketPolicyNone: with empty bucket name empty prefix. + {BucketPolicyNone, "", "hello", `[]`}, + // BucketPolicyNone: with bucket name prefix. + {BucketPolicyNone, "mybucket", "hello", `[]`}, + // BucketPolicyReadOnly: with empty bucket name and prefix. + {BucketPolicyReadOnly, "", "", `[]`}, + // BucketPolicyReadOnly: with bucket name and empty prefix. + {BucketPolicyReadOnly, "mybucket", "", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, + // BucketPolicyReadOnly: with empty bucket name empty prefix. + {BucketPolicyReadOnly, "", "hello", `[]`}, + // BucketPolicyReadOnly: with bucket name prefix. + {BucketPolicyReadOnly, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + // BucketPolicyReadWrite: with empty bucket name and prefix. + {BucketPolicyReadWrite, "", "", `[]`}, + // BucketPolicyReadWrite: with bucket name and empty prefix. + {BucketPolicyReadWrite, "mybucket", "", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, + // BucketPolicyReadWrite: with empty bucket name empty prefix. + {BucketPolicyReadWrite, "", "hello", `[]`}, + // BucketPolicyReadWrite: with bucket name prefix. + {BucketPolicyReadWrite, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + // BucketPolicyWriteOnly: with empty bucket name and prefix. + {BucketPolicyWriteOnly, "", "", `[]`}, + // BucketPolicyWriteOnly: with bucket name and empty prefix. + {BucketPolicyWriteOnly, "mybucket", "", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, + // BucketPolicyWriteOnly: with empty bucket name empty prefix. + {BucketPolicyWriteOnly, "", "hello", `[]`}, + // BucketPolicyWriteOnly: with bucket name prefix. + {BucketPolicyWriteOnly, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + } + + for _, testCase := range testCases { + statements := newStatements(testCase.policy, testCase.bucketName, testCase.prefix) + if data, err := json.Marshal(statements); err == nil { + if string(data) != testCase.expectedResult { + t.Fatalf("%+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) + } + } + } +} + +// getInUsePolicy() is called and the result is validated. +func TestGetInUsePolicy(t *testing.T) { + testCases := []struct { + statements []Statement + bucketName string + prefix string + expectedResult1 bool + expectedResult2 bool + }{ + // All empty statements, bucket name and prefix. + {[]Statement{}, "", "", false, false}, + // Non-empty statements, empty bucket name and empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "", "", false, false}, + // Non-empty statements, non-empty bucket name and empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", false, false}, + // Non-empty statements, empty bucket name and non-empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "", "hello", false, false}, + // Empty statements, non-empty bucket name and empty prefix. + {[]Statement{}, "mybucket", "", false, false}, + // Empty statements, non-empty bucket name non-empty prefix. + {[]Statement{}, "mybucket", "hello", false, false}, + // Empty statements, empty bucket name and non-empty prefix. + {[]Statement{}, "", "hello", false, false}, + // Non-empty statements, non-empty bucket name, non-empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", false, false}, + // different bucket statements and empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), + }}, "mybucket", "", false, false}, + // different bucket statements. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), + }}, "mybucket", "hello", false, false}, + // different bucket multi-statements and empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), + }, { + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::testbucket/world"), + }}, "mybucket", "", false, false}, + // different bucket multi-statements. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), + }, { + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::testbucket/world"), + }}, "mybucket", "hello", false, false}, + // read-only in use. + {[]Statement{{ + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "hello", true, false}, + // write-only in use. + {[]Statement{{ + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "hello", false, true}, + // read-write in use. + {[]Statement{{ + Actions: readWriteObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "hello", true, true}, + // read-write multi-statements. + {[]Statement{{ + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }, { + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/ground"), + }}, "mybucket", "hello", true, true}, + } + + for _, testCase := range testCases { + result1, result2 := getInUsePolicy(testCase.statements, testCase.bucketName, testCase.prefix) + if !(result1 == testCase.expectedResult1 && result2 == testCase.expectedResult2) { + t.Fatalf("%+v: expected: [%t,%t], got: [%t,%t]", testCase, + testCase.expectedResult1, testCase.expectedResult2, + result1, result2) + } + } +} + +// removeStatements() is called and the result is validated. +func TestRemoveStatements(t *testing.T) { + unknownCondMap1 := make(ConditionMap) + unknownCondKeyMap1 := make(ConditionKeyMap) + unknownCondKeyMap1.Add("s3:prefix", set.CreateStringSet("hello")) + unknownCondMap1.Add("StringNotEquals", unknownCondKeyMap1) + + unknownCondMap11 := make(ConditionMap) + unknownCondKeyMap11 := make(ConditionKeyMap) + unknownCondKeyMap11.Add("s3:prefix", set.CreateStringSet("hello")) + unknownCondMap11.Add("StringNotEquals", unknownCondKeyMap11) + + unknownCondMap12 := make(ConditionMap) + unknownCondKeyMap12 := make(ConditionKeyMap) + unknownCondKeyMap12.Add("s3:prefix", set.CreateStringSet("hello")) + unknownCondMap12.Add("StringNotEquals", unknownCondKeyMap12) + + knownCondMap1 := make(ConditionMap) + knownCondKeyMap1 := make(ConditionKeyMap) + knownCondKeyMap1.Add("s3:prefix", set.CreateStringSet("hello")) + knownCondMap1.Add("StringEquals", knownCondKeyMap1) + + knownCondMap11 := make(ConditionMap) + knownCondKeyMap11 := make(ConditionKeyMap) + knownCondKeyMap11.Add("s3:prefix", set.CreateStringSet("hello")) + knownCondMap11.Add("StringEquals", knownCondKeyMap11) + + knownCondMap12 := make(ConditionMap) + knownCondKeyMap12 := make(ConditionKeyMap) + knownCondKeyMap12.Add("s3:prefix", set.CreateStringSet("hello")) + knownCondMap12.Add("StringEquals", knownCondKeyMap12) + + knownCondMap13 := make(ConditionMap) + knownCondKeyMap13 := make(ConditionKeyMap) + knownCondKeyMap13.Add("s3:prefix", set.CreateStringSet("hello")) + knownCondMap13.Add("StringEquals", knownCondKeyMap13) + + knownCondMap14 := make(ConditionMap) + knownCondKeyMap14 := make(ConditionKeyMap) + knownCondKeyMap14.Add("s3:prefix", set.CreateStringSet("hello")) + knownCondMap14.Add("StringEquals", knownCondKeyMap14) + + knownCondMap2 := make(ConditionMap) + knownCondKeyMap2 := make(ConditionKeyMap) + knownCondKeyMap2.Add("s3:prefix", set.CreateStringSet("hello", "world")) + knownCondMap2.Add("StringEquals", knownCondKeyMap2) + + testCases := []struct { + statements []Statement + bucketName string + prefix string + expectedResult string + }{ + // All empty statements, bucket name and prefix. + {[]Statement{}, "", "", `[]`}, + // Non-empty statements, empty bucket name and empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Non-empty statements, non-empty bucket name and empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Non-empty statements, empty bucket name and non-empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "", "hello", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Empty statements, non-empty bucket name and empty prefix. + {[]Statement{}, "mybucket", "", `[]`}, + // Empty statements, non-empty bucket name non-empty prefix. + {[]Statement{}, "mybucket", "hello", `[]`}, + // Empty statements, empty bucket name and non-empty prefix. + {[]Statement{}, "", "hello", `[]`}, + // Statement with unknown Actions with empty prefix. + {[]Statement{{ + Actions: set.CreateStringSet("s3:ListBucketVersions", "s3:ListAllMyBuckets"), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", `[{"Action":["s3:ListAllMyBuckets","s3:ListBucketVersions"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statement with unknown Actions. + {[]Statement{{ + Actions: set.CreateStringSet("s3:ListBucketVersions", "s3:ListAllMyBuckets"), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", `[{"Action":["s3:ListAllMyBuckets","s3:ListBucketVersions"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statement with unknown Effect with empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Deny", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Deny","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statement with unknown Effect. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Deny", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Effect":"Deny","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statement with unknown Principal.User.AWS with empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("arn:aws:iam::AccountNumberWithoutHyphens:root")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["arn:aws:iam::AccountNumberWithoutHyphens:root"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statement with unknown Principal.User.AWS. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("arn:aws:iam::AccountNumberWithoutHyphens:root")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["arn:aws:iam::AccountNumberWithoutHyphens:root"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statement with unknown Principal.User.CanonicalUser with empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{CanonicalUser: set.CreateStringSet("649262f44b8145cb")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"CanonicalUser":["649262f44b8145cb"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statement with unknown Principal.User.CanonicalUser. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{CanonicalUser: set.CreateStringSet("649262f44b8145cb")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"CanonicalUser":["649262f44b8145cb"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statement with unknown Conditions with empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statement with unknown Conditions. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statement with unknown Resource and empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::testbucket"],"Sid":""}]`}, + // Statement with unknown Resource. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::testbucket"],"Sid":""}]`}, + // Statement with known Actions with empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", `[]`}, + // Statement with known Actions. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", `[]`}, + // Statement with known multiple Actions with empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions).Union(commonBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", `[]`}, + // Statement with known multiple Actions. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions).Union(commonBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", `[]`}, + // RemoveBucketActions with readOnlyInUse. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with prefix, readOnlyInUse. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with writeOnlyInUse. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "", `[{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with prefix, writeOnlyInUse. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with readOnlyInUse and writeOnlyInUse. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readWriteObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with prefix, readOnlyInUse and writeOnlyInUse. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readWriteObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with known Conditions, readOnlyInUse. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: knownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with prefix, known Conditions, readOnlyInUse. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: knownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "hello", `[{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with prefix, known Conditions contains other object prefix, readOnlyInUse. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: knownCondMap2, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Condition":{"StringEquals":{"s3:prefix":["world"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with unknown Conditions, readOnlyInUse. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with prefix, unknown Conditions, readOnlyInUse. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with known Conditions, writeOnlyInUse. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: knownCondMap11, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with prefix, known Conditions, writeOnlyInUse. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: knownCondMap11, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with unknown Conditions, writeOnlyInUse. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap11, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with prefix, unknown Conditions, writeOnlyInUse. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap11, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with known Conditions, readOnlyInUse and writeOnlyInUse. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: knownCondMap12, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with prefix, known Conditions, readOnlyInUse and writeOnlyInUse. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: knownCondMap12, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "hello", `[{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with unknown Conditions, readOnlyInUse and writeOnlyInUse. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap12, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // RemoveBucketActions with prefix, unknown Conditions, readOnlyInUse and writeOnlyInUse. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap12, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, + // readOnlyObjectActions - RemoveObjectActions with known condition. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: knownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + // readOnlyObjectActions - RemoveObjectActions with prefix, known condition. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: knownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }}, "mybucket", "hello", `[]`}, + // readOnlyObjectActions - RemoveObjectActions with unknown condition. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + // readOnlyObjectActions - RemoveObjectActions with prefix, unknown condition. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // writeOnlyObjectActions - RemoveObjectActions with known condition. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: knownCondMap13, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }}, "mybucket", "", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + // writeOnlyObjectActions - RemoveObjectActions with prefix, known condition. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: knownCondMap13, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // writeOnlyObjectActions - RemoveObjectActions with unknown condition. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }}, "mybucket", "", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + // writeOnlyObjectActions - RemoveObjectActions with prefix, unknown condition. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // readWriteObjectActions - RemoveObjectActions with known condition. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: knownCondMap14, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + // readWriteObjectActions - RemoveObjectActions with prefix, known condition. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: knownCondMap13, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }}, "mybucket", "hello", `[]`}, + // readWriteObjectActions - RemoveObjectActions with unknown condition. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }}, "mybucket", "", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + // readWriteObjectActions - RemoveObjectActions with prefix, unknown condition. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, { + Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }}, "mybucket", "hello", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + } + + for _, testCase := range testCases { + statements := removeStatements(testCase.statements, testCase.bucketName, testCase.prefix) + if data, err := json.Marshal(statements); err != nil { + t.Fatalf("unable encoding to json, %s", err) + } else if string(data) != testCase.expectedResult { + t.Fatalf("%+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) + } + } +} + +// appendStatement() is called and the result is validated. +func TestAppendStatement(t *testing.T) { + condMap := make(ConditionMap) + condKeyMap := make(ConditionKeyMap) + condKeyMap.Add("s3:prefix", set.CreateStringSet("hello")) + condMap.Add("StringEquals", condKeyMap) + + condMap1 := make(ConditionMap) + condKeyMap1 := make(ConditionKeyMap) + condKeyMap1.Add("s3:prefix", set.CreateStringSet("world")) + condMap1.Add("StringEquals", condKeyMap1) + + unknownCondMap1 := make(ConditionMap) + unknownCondKeyMap1 := make(ConditionKeyMap) + unknownCondKeyMap1.Add("s3:prefix", set.CreateStringSet("world")) + unknownCondMap1.Add("StringNotEquals", unknownCondKeyMap1) + + testCases := []struct { + statements []Statement + statement Statement + expectedResult string + }{ + // Empty statements and empty new statement. + {[]Statement{}, Statement{}, `[]`}, + // Non-empty statements and empty new statement. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, Statement{}, `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Empty statements and non-empty new statement. + {[]Statement{}, Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Append existing statement. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Append same statement with different resource. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), + }, `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket","arn:aws:s3:::testbucket"],"Sid":""}]`}, + // Append same statement with different actions. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Elements of new statement contains elements in statements. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket", "arn:aws:s3:::testbucket"), + }}, Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket","arn:aws:s3:::testbucket"],"Sid":""}]`}, + // Elements of new statement with conditions contains elements in statements. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: condMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket", "arn:aws:s3:::testbucket"), + }}, Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: condMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket","arn:aws:s3:::testbucket"],"Sid":""}]`}, + // Statements with condition and new statement with condition. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: condMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket", "arn:aws:s3:::testbucket"), + }}, Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: condMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket","arn:aws:s3:::testbucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["world"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statements with condition and same resources, and new statement with condition. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: condMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: condMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello","world"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statements with unknown condition and same resources, and new statement with known condition. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: unknownCondMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: condMap1, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["world"]},"StringNotEquals":{"s3:prefix":["world"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statements without condition and new statement with condition. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket", "arn:aws:s3:::testbucket"), + }}, Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: condMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket","arn:aws:s3:::testbucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statements with condition and new statement without condition. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: condMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket", "arn:aws:s3:::testbucket"), + }}, Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket","arn:aws:s3:::testbucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // Statements and new statement are different. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, Statement{ + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }, `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + } + + for _, testCase := range testCases { + statements := appendStatement(testCase.statements, testCase.statement) + if data, err := json.Marshal(statements); err != nil { + t.Fatalf("unable encoding to json, %s", err) + } else if string(data) != testCase.expectedResult { + t.Fatalf("%+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) + } + } +} + +// getBucketPolicy() is called and the result is validated. +func TestGetBucketPolicy(t *testing.T) { + helloCondMap := make(ConditionMap) + helloCondKeyMap := make(ConditionKeyMap) + helloCondKeyMap.Add("s3:prefix", set.CreateStringSet("hello")) + helloCondMap.Add("StringEquals", helloCondKeyMap) + + worldCondMap := make(ConditionMap) + worldCondKeyMap := make(ConditionKeyMap) + worldCondKeyMap.Add("s3:prefix", set.CreateStringSet("world")) + worldCondMap.Add("StringEquals", worldCondKeyMap) + + notHelloCondMap := make(ConditionMap) + notHelloCondMap.Add("StringNotEquals", worldCondKeyMap) + + testCases := []struct { + statement Statement + prefix string + expectedResult1 bool + expectedResult2 bool + expectedResult3 bool + }{ + // Statement with invalid Effect. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Deny", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "", false, false, false}, + // Statement with invalid Effect with prefix. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Deny", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "hello", false, false, false}, + // Statement with invalid Principal.AWS. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("arn:aws:iam::AccountNumberWithoutHyphens:root")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "", false, false, false}, + // Statement with invalid Principal.AWS with prefix. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("arn:aws:iam::AccountNumberWithoutHyphens:root")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "hello", false, false, false}, + + // Statement with commonBucketActions. + {Statement{ + Actions: commonBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "", true, false, false}, + // Statement with commonBucketActions. + {Statement{ + Actions: commonBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "hello", true, false, false}, + + // Statement with commonBucketActions and condition. + {Statement{ + Actions: commonBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "", false, false, false}, + // Statement with commonBucketActions and condition. + {Statement{ + Actions: commonBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "hello", false, false, false}, + // Statement with writeOnlyBucketActions. + {Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "", false, false, true}, + // Statement with writeOnlyBucketActions. + {Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "hello", false, false, true}, + // Statement with writeOnlyBucketActions and condition + {Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "", false, false, false}, + // Statement with writeOnlyBucketActions and condition. + {Statement{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "hello", false, false, false}, + // Statement with readOnlyBucketActions. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "", false, true, false}, + // Statement with readOnlyBucketActions. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "hello", false, true, false}, + // Statement with readOnlyBucketActions with empty condition. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "", false, false, false}, + // Statement with readOnlyBucketActions with empty condition. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "hello", false, false, false}, + // Statement with readOnlyBucketActions with matching condition. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: helloCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "", false, false, false}, + // Statement with readOnlyBucketActions with matching condition. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: helloCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "hello", false, true, false}, + + // Statement with readOnlyBucketActions with different condition. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: worldCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "", false, false, false}, + // Statement with readOnlyBucketActions with different condition. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: worldCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "hello", false, false, false}, + + // Statement with readOnlyBucketActions with StringNotEquals condition. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: notHelloCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "", false, false, false}, + // Statement with readOnlyBucketActions with StringNotEquals condition. + {Statement{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: notHelloCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, "hello", false, true, false}, + } + + for _, testCase := range testCases { + commonFound, readOnly, writeOnly := getBucketPolicy(testCase.statement, testCase.prefix) + if !(testCase.expectedResult1 == commonFound && testCase.expectedResult2 == readOnly && testCase.expectedResult3 == writeOnly) { + t.Fatalf("%+v: expected: [%t,%t,%t], got: [%t,%t,%t]", testCase, + testCase.expectedResult1, testCase.expectedResult2, testCase.expectedResult3, + commonFound, readOnly, writeOnly) + } + } +} + +// getObjectPolicy() is called and the result is validated. +func TestGetObjectPolicy(t *testing.T) { + testCases := []struct { + statement Statement + expectedResult1 bool + expectedResult2 bool + }{ + // Statement with invalid Effect. + {Statement{ + Actions: readOnlyObjectActions, + Effect: "Deny", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }, false, false}, + // Statement with invalid Principal.AWS. + {Statement{ + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("arn:aws:iam::AccountNumberWithoutHyphens:root")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }, false, false}, + // Statement with condition. + {Statement{ + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: make(ConditionMap), + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }, false, false}, + // Statement with readOnlyObjectActions. + {Statement{ + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }, true, false}, + // Statement with writeOnlyObjectActions. + {Statement{ + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }, false, true}, + // Statement with readOnlyObjectActions and writeOnlyObjectActions. + {Statement{ + Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), + }, true, true}, + } + + for _, testCase := range testCases { + readOnly, writeOnly := getObjectPolicy(testCase.statement) + if !(testCase.expectedResult1 == readOnly && testCase.expectedResult2 == writeOnly) { + t.Fatalf("%+v: expected: [%t,%t], got: [%t,%t]", testCase, + testCase.expectedResult1, testCase.expectedResult2, + readOnly, writeOnly) + } + } +} + +// GetPolicyRules is called and the result is validated +func TestListBucketPolicies(t *testing.T) { + + // Condition for read objects + downloadCondMap := make(ConditionMap) + downloadCondKeyMap := make(ConditionKeyMap) + downloadCondKeyMap.Add("s3:prefix", set.CreateStringSet("download")) + downloadCondMap.Add("StringEquals", downloadCondKeyMap) + + // Condition for readwrite objects + downloadUploadCondMap := make(ConditionMap) + downloadUploadCondKeyMap := make(ConditionKeyMap) + downloadUploadCondKeyMap.Add("s3:prefix", set.CreateStringSet("both")) + downloadUploadCondMap.Add("StringEquals", downloadUploadCondKeyMap) + + commonSetActions := commonBucketActions.Union(readOnlyBucketActions) + testCases := []struct { + statements []Statement + bucketName string + prefix string + expectedResult map[string]BucketPolicy + }{ + // Empty statements, bucket name and prefix. + {[]Statement{}, "", "", map[string]BucketPolicy{}}, + // Non-empty statements, empty bucket name and empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "", "", map[string]BucketPolicy{}}, + // Empty statements, non-empty bucket name and empty prefix. + {[]Statement{}, "mybucket", "", map[string]BucketPolicy{}}, + // Readonly object statement + {[]Statement{ + { + Actions: commonBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, + { + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: downloadCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, + { + Actions: readOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/download*"), + }}, "mybucket", "", map[string]BucketPolicy{"mybucket/download*": BucketPolicyReadOnly}}, + {[]Statement{ + { + Actions: commonSetActions.Union(readOnlyObjectActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket", "arn:aws:s3:::mybucket/*"), + }}, "mybucket", "", map[string]BucketPolicy{"mybucket/*": BucketPolicyReadOnly}}, + // Write Only + {[]Statement{ + { + Actions: commonBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, + { + Actions: writeOnlyObjectActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/upload*"), + }}, "mybucket", "", map[string]BucketPolicy{"mybucket/upload*": BucketPolicyWriteOnly}}, + // Readwrite + {[]Statement{ + { + Actions: commonBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, + { + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: downloadUploadCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }, + { + Actions: writeOnlyObjectActions.Union(readOnlyObjectActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket/both*"), + }}, "mybucket", "", map[string]BucketPolicy{"mybucket/both*": BucketPolicyReadWrite}}, + } + + for _, testCase := range testCases { + policyRules := GetPolicies(testCase.statements, testCase.bucketName, "") + if !reflect.DeepEqual(testCase.expectedResult, policyRules) { + t.Fatalf("%+v:\n expected: %+v, got: %+v", testCase, testCase.expectedResult, policyRules) + } + } +} + +// GetPolicy() is called and the result is validated. +func TestGetPolicy(t *testing.T) { + helloCondMap := make(ConditionMap) + helloCondKeyMap := make(ConditionKeyMap) + helloCondKeyMap.Add("s3:prefix", set.CreateStringSet("hello")) + helloCondMap.Add("StringEquals", helloCondKeyMap) + + testCases := []struct { + statements []Statement + bucketName string + prefix string + expectedResult BucketPolicy + }{ + // Empty statements, bucket name and prefix. + {[]Statement{}, "", "", BucketPolicyNone}, + // Non-empty statements, empty bucket name and empty prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "", "", BucketPolicyNone}, + // Empty statements, non-empty bucket name and empty prefix. + {[]Statement{}, "mybucket", "", BucketPolicyNone}, + // not-matching Statements. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), + }}, "mybucket", "", BucketPolicyNone}, + // not-matching Statements with prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), + }}, "mybucket", "hello", BucketPolicyNone}, + // Statements with only commonBucketActions. + {[]Statement{{ + Actions: commonBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", BucketPolicyNone}, + // Statements with only commonBucketActions with prefix. + {[]Statement{{ + Actions: commonBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", BucketPolicyNone}, + // Statements with only readOnlyBucketActions. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", BucketPolicyNone}, + // Statements with only readOnlyBucketActions with prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", BucketPolicyNone}, + // Statements with only readOnlyBucketActions with conditions. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: helloCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", BucketPolicyNone}, + // Statements with only readOnlyBucketActions with prefix with conditons. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: helloCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", BucketPolicyNone}, + // Statements with only writeOnlyBucketActions. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", BucketPolicyNone}, + // Statements with only writeOnlyBucketActions with prefix. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", BucketPolicyNone}, + // Statements with only readOnlyBucketActions + writeOnlyBucketActions. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", BucketPolicyNone}, + // Statements with only readOnlyBucketActions + writeOnlyBucketActions with prefix. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", BucketPolicyNone}, + // Statements with only readOnlyBucketActions + writeOnlyBucketActions and conditions. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: helloCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "", BucketPolicyNone}, + // Statements with only readOnlyBucketActions + writeOnlyBucketActions and conditions with prefix. + {[]Statement{{ + Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: helloCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, "mybucket", "hello", BucketPolicyNone}, + } + + for _, testCase := range testCases { + policy := GetPolicy(testCase.statements, testCase.bucketName, testCase.prefix) + if testCase.expectedResult != policy { + t.Fatalf("%+v: expected: %s, got: %s", testCase, testCase.expectedResult, policy) + } + } +} + +// SetPolicy() is called and the result is validated. +func TestSetPolicy(t *testing.T) { + helloCondMap := make(ConditionMap) + helloCondKeyMap := make(ConditionKeyMap) + helloCondKeyMap.Add("s3:prefix", set.CreateStringSet("hello")) + helloCondMap.Add("StringEquals", helloCondKeyMap) + + testCases := []struct { + statements []Statement + policy BucketPolicy + bucketName string + prefix string + expectedResult string + }{ + // BucketPolicyNone - empty statements, bucket name and prefix. + {[]Statement{}, BucketPolicyNone, "", "", `[]`}, + // BucketPolicyNone - non-empty statements, bucket name and prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, BucketPolicyNone, "", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, + // BucketPolicyNone - empty statements, non-empty bucket name and prefix. + {[]Statement{}, BucketPolicyNone, "mybucket", "", `[]`}, + // BucketPolicyNone - empty statements, bucket name and non-empty prefix. + {[]Statement{}, BucketPolicyNone, "", "hello", `[]`}, + // BucketPolicyReadOnly - empty statements, bucket name and prefix. + {[]Statement{}, BucketPolicyReadOnly, "", "", `[]`}, + // BucketPolicyReadOnly - non-empty statements, bucket name and prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), + }}, BucketPolicyReadOnly, "", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::testbucket"],"Sid":""}]`}, + // BucketPolicyReadOnly - empty statements, non-empty bucket name and prefix. + {[]Statement{}, BucketPolicyReadOnly, "mybucket", "", `[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, + // BucketPolicyReadOnly - empty statements, bucket name and non-empty prefix. + {[]Statement{}, BucketPolicyReadOnly, "", "hello", `[]`}, + // BucketPolicyReadOnly - empty statements, non-empty bucket name and non-empty prefix. + {[]Statement{}, BucketPolicyReadOnly, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + // BucketPolicyWriteOnly - empty statements, bucket name and prefix. + {[]Statement{}, BucketPolicyReadOnly, "", "", `[]`}, + // BucketPolicyWriteOnly - non-empty statements, bucket name and prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), + }}, BucketPolicyWriteOnly, "", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::testbucket"],"Sid":""}]`}, + // BucketPolicyWriteOnly - empty statements, non-empty bucket name and prefix. + {[]Statement{}, BucketPolicyWriteOnly, "mybucket", "", `[{"Action":["s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, + // BucketPolicyWriteOnly - empty statements, bucket name and non-empty prefix. + {[]Statement{}, BucketPolicyWriteOnly, "", "hello", `[]`}, + // BucketPolicyWriteOnly - empty statements, non-empty bucket name and non-empty prefix. + {[]Statement{}, BucketPolicyWriteOnly, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + // BucketPolicyReadWrite - empty statements, bucket name and prefix. + {[]Statement{}, BucketPolicyReadWrite, "", "", `[]`}, + // BucketPolicyReadWrite - non-empty statements, bucket name and prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), + }}, BucketPolicyReadWrite, "", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::testbucket"],"Sid":""}]`}, + // BucketPolicyReadWrite - empty statements, non-empty bucket name and prefix. + {[]Statement{}, BucketPolicyReadWrite, "mybucket", "", `[{"Action":["s3:GetBucketLocation","s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, + // BucketPolicyReadWrite - empty statements, bucket name and non-empty prefix. + {[]Statement{}, BucketPolicyReadWrite, "", "hello", `[]`}, + // BucketPolicyReadWrite - empty statements, non-empty bucket name and non-empty prefix. + {[]Statement{}, BucketPolicyReadWrite, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + // Set readonly. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, BucketPolicyReadOnly, "mybucket", "", `[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, + // Set readonly with prefix. + {[]Statement{{ + Actions: writeOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, BucketPolicyReadOnly, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + // Set writeonly. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, BucketPolicyWriteOnly, "mybucket", "", `[{"Action":["s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, + // Set writeonly with prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: helloCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, BucketPolicyWriteOnly, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + + // Set readwrite. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, BucketPolicyReadWrite, "mybucket", "", `[{"Action":["s3:GetBucketLocation","s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, + // Set readwrite with prefix. + {[]Statement{{ + Actions: readOnlyBucketActions, + Effect: "Allow", + Principal: User{AWS: set.CreateStringSet("*")}, + Conditions: helloCondMap, + Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), + }}, BucketPolicyReadWrite, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, + } + + for _, testCase := range testCases { + statements := SetPolicy(testCase.statements, testCase.policy, testCase.bucketName, testCase.prefix) + if data, err := json.Marshal(statements); err != nil { + t.Fatalf("unable encoding to json, %s", err) + } else if string(data) != testCase.expectedResult { + t.Fatalf("%+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) + } + } +} + +// Validates bucket policy string. +func TestIsValidBucketPolicy(t *testing.T) { + testCases := []struct { + inputPolicy BucketPolicy + expectedResult bool + }{ + // valid inputs. + {BucketPolicy("none"), true}, + {BucketPolicy("readonly"), true}, + {BucketPolicy("readwrite"), true}, + {BucketPolicy("writeonly"), true}, + // invalid input. + {BucketPolicy("readwriteonly"), false}, + {BucketPolicy("writeread"), false}, + } + + for i, testCase := range testCases { + actualResult := testCase.inputPolicy.IsValidBucketPolicy() + if testCase.expectedResult != actualResult { + t.Errorf("Test %d: Expected IsValidBucket policy to be '%v' for policy \"%s\", but instead found it to be '%v'", i+1, testCase.expectedResult, testCase.inputPolicy, actualResult) + } + } +} + +// Tests validate Bucket policy resource matcher. +func TestBucketPolicyResourceMatch(t *testing.T) { + + // generates\ statement with given resource.. + generateStatement := func(resource string) Statement { + statement := Statement{} + statement.Resources = set.CreateStringSet(resource) + return statement + } + + // generates resource prefix. + generateResource := func(bucketName, objectName string) string { + return awsResourcePrefix + bucketName + "/" + objectName + } + + testCases := []struct { + resourceToMatch string + statement Statement + expectedResourceMatch bool + }{ + // Test case 1-4. + // Policy with resource ending with bucket/* allows access to all objects inside the given bucket. + {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/*")), true}, + {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/*")), true}, + {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/*")), true}, + {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/*")), true}, + // Test case - 5. + // Policy with resource ending with bucket/oo* should not allow access to bucket/output.txt. + {generateResource("minio-bucket", "output.txt"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/oo*")), false}, + // Test case - 6. + // Policy with resource ending with bucket/oo* should allow access to bucket/ootput.txt. + {generateResource("minio-bucket", "ootput.txt"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/oo*")), true}, + // Test case - 7. + // Policy with resource ending with bucket/oo* allows access to all subfolders starting with "oo" inside given bucket. + {generateResource("minio-bucket", "oop-bucket/my-file"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/oo*")), true}, + // Test case - 8. + {generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false}, + // Test case - 9. + {generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false}, + // Test case - 10. + // Proves that the name space is flat. + {generateResource("minio-bucket", "Africa/Bihar/India/design_info.doc/Bihar"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, + "minio-bucket"+"/*/India/*/Bihar")), true}, + // Test case - 11. + // Proves that the name space is flat. + {generateResource("minio-bucket", "Asia/China/India/States/Bihar/output.txt"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, + "minio-bucket"+"/*/India/*/Bihar/*")), true}, + } + for i, testCase := range testCases { + resources := testCase.statement.Resources.FuncMatch(resourceMatch, testCase.resourceToMatch) + actualResourceMatch := resources.Equals(testCase.statement.Resources) + if testCase.expectedResourceMatch != actualResourceMatch { + t.Errorf("Test %d: Expected Resource match to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResourceMatch, actualResourceMatch) + } + } +} diff --git a/pkg/s3signer/request-signature-streaming.go b/pkg/s3signer/request-signature-streaming.go new file mode 100644 index 0000000..156a6d6 --- /dev/null +++ b/pkg/s3signer/request-signature-streaming.go @@ -0,0 +1,306 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "bytes" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "net/http" + "strconv" + "strings" + "time" +) + +// Reference for constants used below - +// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming +const ( + streamingSignAlgorithm = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + streamingPayloadHdr = "AWS4-HMAC-SHA256-PAYLOAD" + emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + payloadChunkSize = 64 * 1024 + chunkSigConstLen = 17 // ";chunk-signature=" + signatureStrLen = 64 // e.g. "f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2" + crlfLen = 2 // CRLF +) + +// Request headers to be ignored while calculating seed signature for +// a request. +var ignoredStreamingHeaders = map[string]bool{ + "Authorization": true, + "User-Agent": true, + "Content-Type": true, +} + +// getSignedChunkLength - calculates the length of chunk metadata +func getSignedChunkLength(chunkDataSize int64) int64 { + return int64(len(fmt.Sprintf("%x", chunkDataSize))) + + chunkSigConstLen + + signatureStrLen + + crlfLen + + chunkDataSize + + crlfLen +} + +// getStreamLength - calculates the length of the overall stream (data + metadata) +func getStreamLength(dataLen, chunkSize int64) int64 { + if dataLen <= 0 { + return 0 + } + + chunksCount := int64(dataLen / chunkSize) + remainingBytes := int64(dataLen % chunkSize) + streamLen := int64(0) + streamLen += chunksCount * getSignedChunkLength(chunkSize) + if remainingBytes > 0 { + streamLen += getSignedChunkLength(remainingBytes) + } + streamLen += getSignedChunkLength(0) + return streamLen +} + +// buildChunkStringToSign - returns the string to sign given chunk data +// and previous signature. +func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string { + stringToSignParts := []string{ + streamingPayloadHdr, + t.Format(iso8601DateFormat), + getScope(region, t), + previousSig, + emptySHA256, + hex.EncodeToString(sum256(chunkData)), + } + + return strings.Join(stringToSignParts, "\n") +} + +// prepareStreamingRequest - prepares a request with appropriate +// headers before computing the seed signature. +func prepareStreamingRequest(req *http.Request, sessionToken string, dataLen int64, timestamp time.Time) { + // Set x-amz-content-sha256 header. + req.Header.Set("X-Amz-Content-Sha256", streamingSignAlgorithm) + if sessionToken != "" { + req.Header.Set("X-Amz-Security-Token", sessionToken) + } + + req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat)) + // Set content length with streaming signature for each chunk included. + req.ContentLength = getStreamLength(dataLen, int64(payloadChunkSize)) + req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(dataLen, 10)) +} + +// buildChunkHeader - returns the chunk header. +// e.g string(IntHexBase(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n +func buildChunkHeader(chunkLen int64, signature string) []byte { + return []byte(strconv.FormatInt(chunkLen, 16) + ";chunk-signature=" + signature + "\r\n") +} + +// buildChunkSignature - returns chunk signature for a given chunk and previous signature. +func buildChunkSignature(chunkData []byte, reqTime time.Time, region, + previousSignature, secretAccessKey string) string { + + chunkStringToSign := buildChunkStringToSign(reqTime, region, + previousSignature, chunkData) + signingKey := getSigningKey(secretAccessKey, region, reqTime) + return getSignature(signingKey, chunkStringToSign) +} + +// getSeedSignature - returns the seed signature for a given request. +func (s *StreamingReader) setSeedSignature(req *http.Request) { + // Get canonical request + canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders) + + // Get string to sign from canonical request. + stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest) + + signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime) + + // Calculate signature. + s.seedSignature = getSignature(signingKey, stringToSign) +} + +// StreamingReader implements chunked upload signature as a reader on +// top of req.Body's ReaderCloser chunk header;data;... repeat +type StreamingReader struct { + accessKeyID string + secretAccessKey string + sessionToken string + region string + prevSignature string + seedSignature string + contentLen int64 // Content-Length from req header + baseReadCloser io.ReadCloser // underlying io.Reader + bytesRead int64 // bytes read from underlying io.Reader + buf bytes.Buffer // holds signed chunk + chunkBuf []byte // holds raw data read from req Body + chunkBufLen int // no. of bytes read so far into chunkBuf + done bool // done reading the underlying reader to EOF + reqTime time.Time + chunkNum int + totalChunks int + lastChunkSize int +} + +// signChunk - signs a chunk read from s.baseReader of chunkLen size. +func (s *StreamingReader) signChunk(chunkLen int) { + // Compute chunk signature for next header + signature := buildChunkSignature(s.chunkBuf[:chunkLen], s.reqTime, + s.region, s.prevSignature, s.secretAccessKey) + + // For next chunk signature computation + s.prevSignature = signature + + // Write chunk header into streaming buffer + chunkHdr := buildChunkHeader(int64(chunkLen), signature) + s.buf.Write(chunkHdr) + + // Write chunk data into streaming buffer + s.buf.Write(s.chunkBuf[:chunkLen]) + + // Write the chunk trailer. + s.buf.Write([]byte("\r\n")) + + // Reset chunkBufLen for next chunk read. + s.chunkBufLen = 0 + s.chunkNum++ +} + +// setStreamingAuthHeader - builds and sets authorization header value +// for streaming signature. +func (s *StreamingReader) setStreamingAuthHeader(req *http.Request) { + credential := GetCredential(s.accessKeyID, s.region, s.reqTime) + authParts := []string{ + signV4Algorithm + " Credential=" + credential, + "SignedHeaders=" + getSignedHeaders(*req, ignoredStreamingHeaders), + "Signature=" + s.seedSignature, + } + + // Set authorization header. + auth := strings.Join(authParts, ",") + req.Header.Set("Authorization", auth) +} + +// StreamingSignV4 - provides chunked upload signatureV4 support by +// implementing io.Reader. +func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionToken, + region string, dataLen int64, reqTime time.Time) *http.Request { + + // Set headers needed for streaming signature. + prepareStreamingRequest(req, sessionToken, dataLen, reqTime) + + if req.Body == nil { + req.Body = ioutil.NopCloser(bytes.NewReader([]byte(""))) + } + + stReader := &StreamingReader{ + baseReadCloser: req.Body, + accessKeyID: accessKeyID, + secretAccessKey: secretAccessKey, + sessionToken: sessionToken, + region: region, + reqTime: reqTime, + chunkBuf: make([]byte, payloadChunkSize), + contentLen: dataLen, + chunkNum: 1, + totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1, + lastChunkSize: int(dataLen % payloadChunkSize), + } + + // Add the request headers required for chunk upload signing. + + // Compute the seed signature. + stReader.setSeedSignature(req) + + // Set the authorization header with the seed signature. + stReader.setStreamingAuthHeader(req) + + // Set seed signature as prevSignature for subsequent + // streaming signing process. + stReader.prevSignature = stReader.seedSignature + req.Body = stReader + + return req +} + +// Read - this method performs chunk upload signature providing a +// io.Reader interface. +func (s *StreamingReader) Read(buf []byte) (int, error) { + switch { + // After the last chunk is read from underlying reader, we + // never re-fill s.buf. + case s.done: + + // s.buf will be (re-)filled with next chunk when has lesser + // bytes than asked for. + case s.buf.Len() < len(buf): + s.chunkBufLen = 0 + for { + n1, err := s.baseReadCloser.Read(s.chunkBuf[s.chunkBufLen:]) + // Usually we validate `err` first, but in this case + // we are validating n > 0 for the following reasons. + // + // 1. n > 0, err is one of io.EOF, nil (near end of stream) + // A Reader returning a non-zero number of bytes at the end + // of the input stream may return either err == EOF or err == nil + // + // 2. n == 0, err is io.EOF (actual end of stream) + // + // Callers should always process the n > 0 bytes returned + // before considering the error err. + if n1 > 0 { + s.chunkBufLen += n1 + s.bytesRead += int64(n1) + + if s.chunkBufLen == payloadChunkSize || + (s.chunkNum == s.totalChunks-1 && + s.chunkBufLen == s.lastChunkSize) { + // Sign the chunk and write it to s.buf. + s.signChunk(s.chunkBufLen) + break + } + } + if err != nil { + if err == io.EOF { + // No more data left in baseReader - last chunk. + // Done reading the last chunk from baseReader. + s.done = true + + // bytes read from baseReader different than + // content length provided. + if s.bytesRead != s.contentLen { + return 0, io.ErrUnexpectedEOF + } + + // Sign the chunk and write it to s.buf. + s.signChunk(0) + break + } + return 0, err + } + + } + } + return s.buf.Read(buf) +} + +// Close - this method makes underlying io.ReadCloser's Close method available. +func (s *StreamingReader) Close() error { + return s.baseReadCloser.Close() +} diff --git a/pkg/s3signer/request-signature-streaming_test.go b/pkg/s3signer/request-signature-streaming_test.go new file mode 100644 index 0000000..297ab97 --- /dev/null +++ b/pkg/s3signer/request-signature-streaming_test.go @@ -0,0 +1,109 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "bytes" + "io/ioutil" + "testing" + "time" +) + +func TestGetSeedSignature(t *testing.T) { + accessKeyID := "AKIAIOSFODNN7EXAMPLE" + secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + dataLen := 66560 + data := bytes.Repeat([]byte("a"), dataLen) + body := ioutil.NopCloser(bytes.NewReader(data)) + + req := NewRequest("PUT", "/examplebucket/chunkObject.txt", body) + req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + req.Host = "s3.amazonaws.com" + + reqTime, err := time.Parse("20060102T150405Z", "20130524T000000Z") + if err != nil { + t.Fatalf("Failed to parse time - %v", err) + } + + req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", "us-east-1", int64(dataLen), reqTime) + actualSeedSignature := req.Body.(*StreamingReader).seedSignature + + expectedSeedSignature := "38cab3af09aa15ddf29e26e36236f60fb6bfb6243a20797ae9a8183674526079" + if actualSeedSignature != expectedSeedSignature { + t.Errorf("Expected %s but received %s", expectedSeedSignature, actualSeedSignature) + } +} + +func TestChunkSignature(t *testing.T) { + chunkData := bytes.Repeat([]byte("a"), 65536) + reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z") + previousSignature := "4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9" + location := "us-east-1" + secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + expectedSignature := "ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648" + actualSignature := buildChunkSignature(chunkData, reqTime, location, previousSignature, secretAccessKeyID) + if actualSignature != expectedSignature { + t.Errorf("Expected %s but received %s", expectedSignature, actualSignature) + } +} + +func TestSetStreamingAuthorization(t *testing.T) { + location := "us-east-1" + secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + accessKeyID := "AKIAIOSFODNN7EXAMPLE" + + req := NewRequest("PUT", "/examplebucket/chunkObject.txt", nil) + req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + req.Host = "" + req.URL.Host = "s3.amazonaws.com" + + dataLen := int64(65 * 1024) + reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z") + req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime) + + expectedAuthorization := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=38cab3af09aa15ddf29e26e36236f60fb6bfb6243a20797ae9a8183674526079" + + actualAuthorization := req.Header.Get("Authorization") + if actualAuthorization != expectedAuthorization { + t.Errorf("Expected %s but received %s", expectedAuthorization, actualAuthorization) + } +} + +func TestStreamingReader(t *testing.T) { + reqTime, _ := time.Parse("20060102T150405Z", "20130524T000000Z") + location := "us-east-1" + secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + accessKeyID := "AKIAIOSFODNN7EXAMPLE" + dataLen := int64(65 * 1024) + + req := NewRequest("PUT", "/examplebucket/chunkObject.txt", nil) + req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + req.ContentLength = 65 * 1024 + req.Host = "" + req.URL.Host = "s3.amazonaws.com" + + baseReader := ioutil.NopCloser(bytes.NewReader(bytes.Repeat([]byte("a"), 65*1024))) + req.Body = baseReader + req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime) + + b, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Errorf("Expected no error but received %v %d", err, len(b)) + } + req.Body.Close() +} diff --git a/pkg/s3signer/request-signature-v2.go b/pkg/s3signer/request-signature-v2.go new file mode 100644 index 0000000..b407093 --- /dev/null +++ b/pkg/s3signer/request-signature-v2.go @@ -0,0 +1,316 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "bytes" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "fmt" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// Signature and API related constants. +const ( + signV2Algorithm = "AWS" +) + +// Encode input URL path to URL encoded path. +func encodeURL2Path(req *http.Request, virtualHost bool) (path string) { + if virtualHost { + reqHost := getHostAddr(req) + dotPos := strings.Index(reqHost, ".") + if dotPos > -1 { + bucketName := reqHost[:dotPos] + path = "/" + bucketName + path += req.URL.Path + path = s3utils.EncodePath(path) + return + } + } + path = s3utils.EncodePath(req.URL.Path) + return +} + +// PreSignV2 - presign the request in following style. +// https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}. +func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires int64, virtualHost bool) *http.Request { + // Presign is not needed for anonymous credentials. + if accessKeyID == "" || secretAccessKey == "" { + return &req + } + + d := time.Now().UTC() + // Find epoch expires when the request will expire. + epochExpires := d.Unix() + expires + + // Add expires header if not present. + if expiresStr := req.Header.Get("Expires"); expiresStr == "" { + req.Header.Set("Expires", strconv.FormatInt(epochExpires, 10)) + } + + // Get presigned string to sign. + stringToSign := preStringToSignV2(req, virtualHost) + hm := hmac.New(sha1.New, []byte(secretAccessKey)) + hm.Write([]byte(stringToSign)) + + // Calculate signature. + signature := base64.StdEncoding.EncodeToString(hm.Sum(nil)) + + query := req.URL.Query() + // Handle specially for Google Cloud Storage. + if strings.Contains(getHostAddr(&req), ".storage.googleapis.com") { + query.Set("GoogleAccessId", accessKeyID) + } else { + query.Set("AWSAccessKeyId", accessKeyID) + } + + // Fill in Expires for presigned query. + query.Set("Expires", strconv.FormatInt(epochExpires, 10)) + + // Encode query and save. + req.URL.RawQuery = s3utils.QueryEncode(query) + + // Save signature finally. + req.URL.RawQuery += "&Signature=" + s3utils.EncodePath(signature) + + // Return. + return &req +} + +// PostPresignSignatureV2 - presigned signature for PostPolicy +// request. +func PostPresignSignatureV2(policyBase64, secretAccessKey string) string { + hm := hmac.New(sha1.New, []byte(secretAccessKey)) + hm.Write([]byte(policyBase64)) + signature := base64.StdEncoding.EncodeToString(hm.Sum(nil)) + return signature +} + +// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature; +// Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID, UTF-8-Encoding-Of( StringToSign ) ) ); +// +// StringToSign = HTTP-Verb + "\n" + +// Content-Md5 + "\n" + +// Content-Type + "\n" + +// Date + "\n" + +// CanonicalizedProtocolHeaders + +// CanonicalizedResource; +// +// CanonicalizedResource = [ "/" + Bucket ] + +// + +// [ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; +// +// CanonicalizedProtocolHeaders = + +// SignV2 sign the request before Do() (AWS Signature Version 2). +func SignV2(req http.Request, accessKeyID, secretAccessKey string, virtualHost bool) *http.Request { + // Signature calculation is not needed for anonymous credentials. + if accessKeyID == "" || secretAccessKey == "" { + return &req + } + + // Initial time. + d := time.Now().UTC() + + // Add date if not present. + if date := req.Header.Get("Date"); date == "" { + req.Header.Set("Date", d.Format(http.TimeFormat)) + } + + // Calculate HMAC for secretAccessKey. + stringToSign := stringToSignV2(req, virtualHost) + hm := hmac.New(sha1.New, []byte(secretAccessKey)) + hm.Write([]byte(stringToSign)) + + // Prepare auth header. + authHeader := new(bytes.Buffer) + authHeader.WriteString(fmt.Sprintf("%s %s:", signV2Algorithm, accessKeyID)) + encoder := base64.NewEncoder(base64.StdEncoding, authHeader) + encoder.Write(hm.Sum(nil)) + encoder.Close() + + // Set Authorization header. + req.Header.Set("Authorization", authHeader.String()) + + return &req +} + +// From the Amazon docs: +// +// StringToSign = HTTP-Verb + "\n" + +// Content-Md5 + "\n" + +// Content-Type + "\n" + +// Expires + "\n" + +// CanonicalizedProtocolHeaders + +// CanonicalizedResource; +func preStringToSignV2(req http.Request, virtualHost bool) string { + buf := new(bytes.Buffer) + // Write standard headers. + writePreSignV2Headers(buf, req) + // Write canonicalized protocol headers if any. + writeCanonicalizedHeaders(buf, req) + // Write canonicalized Query resources if any. + writeCanonicalizedResource(buf, req, virtualHost) + return buf.String() +} + +// writePreSignV2Headers - write preSign v2 required headers. +func writePreSignV2Headers(buf *bytes.Buffer, req http.Request) { + buf.WriteString(req.Method + "\n") + buf.WriteString(req.Header.Get("Content-Md5") + "\n") + buf.WriteString(req.Header.Get("Content-Type") + "\n") + buf.WriteString(req.Header.Get("Expires") + "\n") +} + +// From the Amazon docs: +// +// StringToSign = HTTP-Verb + "\n" + +// Content-Md5 + "\n" + +// Content-Type + "\n" + +// Date + "\n" + +// CanonicalizedProtocolHeaders + +// CanonicalizedResource; +func stringToSignV2(req http.Request, virtualHost bool) string { + buf := new(bytes.Buffer) + // Write standard headers. + writeSignV2Headers(buf, req) + // Write canonicalized protocol headers if any. + writeCanonicalizedHeaders(buf, req) + // Write canonicalized Query resources if any. + writeCanonicalizedResource(buf, req, virtualHost) + return buf.String() +} + +// writeSignV2Headers - write signV2 required headers. +func writeSignV2Headers(buf *bytes.Buffer, req http.Request) { + buf.WriteString(req.Method + "\n") + buf.WriteString(req.Header.Get("Content-Md5") + "\n") + buf.WriteString(req.Header.Get("Content-Type") + "\n") + buf.WriteString(req.Header.Get("Date") + "\n") +} + +// writeCanonicalizedHeaders - write canonicalized headers. +func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) { + var protoHeaders []string + vals := make(map[string][]string) + for k, vv := range req.Header { + // All the AMZ headers should be lowercase + lk := strings.ToLower(k) + if strings.HasPrefix(lk, "x-amz") { + protoHeaders = append(protoHeaders, lk) + vals[lk] = vv + } + } + sort.Strings(protoHeaders) + for _, k := range protoHeaders { + buf.WriteString(k) + buf.WriteByte(':') + for idx, v := range vals[k] { + if idx > 0 { + buf.WriteByte(',') + } + if strings.Contains(v, "\n") { + // TODO: "Unfold" long headers that + // span multiple lines (as allowed by + // RFC 2616, section 4.2) by replacing + // the folding white-space (including + // new-line) by a single space. + buf.WriteString(v) + } else { + buf.WriteString(v) + } + } + buf.WriteByte('\n') + } +} + +// AWS S3 Signature V2 calculation rule is give here: +// http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationStringToSign + +// Whitelist resource list that will be used in query string for signature-V2 calculation. +// The list should be alphabetically sorted +var resourceList = []string{ + "acl", + "delete", + "lifecycle", + "location", + "logging", + "notification", + "partNumber", + "policy", + "requestPayment", + "response-cache-control", + "response-content-disposition", + "response-content-encoding", + "response-content-language", + "response-content-type", + "response-expires", + "torrent", + "uploadId", + "uploads", + "versionId", + "versioning", + "versions", + "website", +} + +// From the Amazon docs: +// +// CanonicalizedResource = [ "/" + Bucket ] + +// + +// [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; +func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request, virtualHost bool) { + // Save request URL. + requestURL := req.URL + // Get encoded URL path. + buf.WriteString(encodeURL2Path(&req, virtualHost)) + if requestURL.RawQuery != "" { + var n int + vals, _ := url.ParseQuery(requestURL.RawQuery) + // Verify if any sub resource queries are present, if yes + // canonicallize them. + for _, resource := range resourceList { + if vv, ok := vals[resource]; ok && len(vv) > 0 { + n++ + // First element + switch n { + case 1: + buf.WriteByte('?') + // The rest + default: + buf.WriteByte('&') + } + buf.WriteString(resource) + // Request parameters + if len(vv[0]) > 0 { + buf.WriteByte('=') + buf.WriteString(vv[0]) + } + } + } + } +} diff --git a/pkg/s3signer/request-signature-v2_test.go b/pkg/s3signer/request-signature-v2_test.go new file mode 100644 index 0000000..042b6e6 --- /dev/null +++ b/pkg/s3signer/request-signature-v2_test.go @@ -0,0 +1,36 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "sort" + "testing" +) + +// Tests for 'func TestResourceListSorting(t *testing.T)'. +func TestResourceListSorting(t *testing.T) { + sortedResourceList := make([]string, len(resourceList)) + copy(sortedResourceList, resourceList) + sort.Strings(sortedResourceList) + for i := 0; i < len(resourceList); i++ { + if resourceList[i] != sortedResourceList[i] { + t.Errorf("Expected resourceList[%d] = \"%s\", resourceList is not correctly sorted.", i, sortedResourceList[i]) + break + } + } +} diff --git a/pkg/s3signer/request-signature-v4.go b/pkg/s3signer/request-signature-v4.go new file mode 100644 index 0000000..daf02fe --- /dev/null +++ b/pkg/s3signer/request-signature-v4.go @@ -0,0 +1,315 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "bytes" + "encoding/hex" + "net/http" + "sort" + "strconv" + "strings" + "time" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// Signature and API related constants. +const ( + signV4Algorithm = "AWS4-HMAC-SHA256" + iso8601DateFormat = "20060102T150405Z" + yyyymmdd = "20060102" +) + +/// +/// Excerpts from @lsegal - +/// https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258. +/// +/// User-Agent: +/// +/// This is ignored from signing because signing this causes +/// problems with generating pre-signed URLs (that are executed +/// by other agents) or when customers pass requests through +/// proxies, which may modify the user-agent. +/// +/// Content-Length: +/// +/// This is ignored from signing because generating a pre-signed +/// URL should not provide a content-length constraint, +/// specifically when vending a S3 pre-signed PUT URL. The +/// corollary to this is that when sending regular requests +/// (non-pre-signed), the signature contains a checksum of the +/// body, which implicitly validates the payload length (since +/// changing the number of bytes would change the checksum) +/// and therefore this header is not valuable in the signature. +/// +/// Content-Type: +/// +/// Signing this header causes quite a number of problems in +/// browser environments, where browsers like to modify and +/// normalize the content-type header in different ways. There is +/// more information on this in https://goo.gl/2E9gyy. Avoiding +/// this field simplifies logic and reduces the possibility of +/// future bugs. +/// +/// Authorization: +/// +/// Is skipped for obvious reasons +/// +var v4IgnoredHeaders = map[string]bool{ + "Authorization": true, + "Content-Type": true, + "Content-Length": true, + "User-Agent": true, +} + +// getSigningKey hmac seed to calculate final signature. +func getSigningKey(secret, loc string, t time.Time) []byte { + date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd))) + location := sumHMAC(date, []byte(loc)) + service := sumHMAC(location, []byte("s3")) + signingKey := sumHMAC(service, []byte("aws4_request")) + return signingKey +} + +// getSignature final signature in hexadecimal form. +func getSignature(signingKey []byte, stringToSign string) string { + return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) +} + +// getScope generate a string of a specific date, an AWS region, and a +// service. +func getScope(location string, t time.Time) string { + scope := strings.Join([]string{ + t.Format(yyyymmdd), + location, + "s3", + "aws4_request", + }, "/") + return scope +} + +// GetCredential generate a credential string. +func GetCredential(accessKeyID, location string, t time.Time) string { + scope := getScope(location, t) + return accessKeyID + "/" + scope +} + +// getHashedPayload get the hexadecimal value of the SHA256 hash of +// the request payload. +func getHashedPayload(req http.Request) string { + hashedPayload := req.Header.Get("X-Amz-Content-Sha256") + if hashedPayload == "" { + // Presign does not have a payload, use S3 recommended value. + hashedPayload = unsignedPayload + } + return hashedPayload +} + +// getCanonicalHeaders generate a list of request headers for +// signature. +func getCanonicalHeaders(req http.Request, ignoredHeaders map[string]bool) string { + var headers []string + vals := make(map[string][]string) + for k, vv := range req.Header { + if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok { + continue // ignored header + } + headers = append(headers, strings.ToLower(k)) + vals[strings.ToLower(k)] = vv + } + headers = append(headers, "host") + sort.Strings(headers) + + var buf bytes.Buffer + // Save all the headers in canonical form
: newline + // separated for each header. + for _, k := range headers { + buf.WriteString(k) + buf.WriteByte(':') + switch { + case k == "host": + buf.WriteString(getHostAddr(&req)) + fallthrough + default: + for idx, v := range vals[k] { + if idx > 0 { + buf.WriteByte(',') + } + buf.WriteString(v) + } + buf.WriteByte('\n') + } + } + return buf.String() +} + +// getSignedHeaders generate all signed request headers. +// i.e lexically sorted, semicolon-separated list of lowercase +// request header names. +func getSignedHeaders(req http.Request, ignoredHeaders map[string]bool) string { + var headers []string + for k := range req.Header { + if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok { + continue // Ignored header found continue. + } + headers = append(headers, strings.ToLower(k)) + } + headers = append(headers, "host") + sort.Strings(headers) + return strings.Join(headers, ";") +} + +// getCanonicalRequest generate a canonical request of style. +// +// canonicalRequest = +// \n +// \n +// \n +// \n +// \n +// +func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool) string { + req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1) + canonicalRequest := strings.Join([]string{ + req.Method, + s3utils.EncodePath(req.URL.Path), + req.URL.RawQuery, + getCanonicalHeaders(req, ignoredHeaders), + getSignedHeaders(req, ignoredHeaders), + getHashedPayload(req), + }, "\n") + return canonicalRequest +} + +// getStringToSign a string based on selected query values. +func getStringToSignV4(t time.Time, location, canonicalRequest string) string { + stringToSign := signV4Algorithm + "\n" + t.Format(iso8601DateFormat) + "\n" + stringToSign = stringToSign + getScope(location, t) + "\n" + stringToSign = stringToSign + hex.EncodeToString(sum256([]byte(canonicalRequest))) + return stringToSign +} + +// PreSignV4 presign the request, in accordance with +// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html. +func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, expires int64) *http.Request { + // Presign is not needed for anonymous credentials. + if accessKeyID == "" || secretAccessKey == "" { + return &req + } + + // Initial time. + t := time.Now().UTC() + + // Get credential string. + credential := GetCredential(accessKeyID, location, t) + + // Get all signed headers. + signedHeaders := getSignedHeaders(req, v4IgnoredHeaders) + + // Set URL query. + query := req.URL.Query() + query.Set("X-Amz-Algorithm", signV4Algorithm) + query.Set("X-Amz-Date", t.Format(iso8601DateFormat)) + query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10)) + query.Set("X-Amz-SignedHeaders", signedHeaders) + query.Set("X-Amz-Credential", credential) + // Set session token if available. + if sessionToken != "" { + query.Set("X-Amz-Security-Token", sessionToken) + } + req.URL.RawQuery = query.Encode() + + // Get canonical request. + canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders) + + // Get string to sign from canonical request. + stringToSign := getStringToSignV4(t, location, canonicalRequest) + + // Gext hmac signing key. + signingKey := getSigningKey(secretAccessKey, location, t) + + // Calculate signature. + signature := getSignature(signingKey, stringToSign) + + // Add signature header to RawQuery. + req.URL.RawQuery += "&X-Amz-Signature=" + signature + + return &req +} + +// PostPresignSignatureV4 - presigned signature for PostPolicy +// requests. +func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string { + // Get signining key. + signingkey := getSigningKey(secretAccessKey, location, t) + // Calculate signature. + signature := getSignature(signingkey, policyBase64) + return signature +} + +// SignV4 sign the request before Do(), in accordance with +// http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html. +func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request { + // Signature calculation is not needed for anonymous credentials. + if accessKeyID == "" || secretAccessKey == "" { + return &req + } + + // Initial time. + t := time.Now().UTC() + + // Set x-amz-date. + req.Header.Set("X-Amz-Date", t.Format(iso8601DateFormat)) + + // Set session token if available. + if sessionToken != "" { + req.Header.Set("X-Amz-Security-Token", sessionToken) + } + + // Get canonical request. + canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders) + + // Get string to sign from canonical request. + stringToSign := getStringToSignV4(t, location, canonicalRequest) + + // Get hmac signing key. + signingKey := getSigningKey(secretAccessKey, location, t) + + // Get credential string. + credential := GetCredential(accessKeyID, location, t) + + // Get all signed headers. + signedHeaders := getSignedHeaders(req, v4IgnoredHeaders) + + // Calculate signature. + signature := getSignature(signingKey, stringToSign) + + // If regular request, construct the final authorization header. + parts := []string{ + signV4Algorithm + " Credential=" + credential, + "SignedHeaders=" + signedHeaders, + "Signature=" + signature, + } + + // Set authorization header. + auth := strings.Join(parts, ", ") + req.Header.Set("Authorization", auth) + + return &req +} diff --git a/pkg/s3signer/request-signature-v4_test.go b/pkg/s3signer/request-signature-v4_test.go new file mode 100644 index 0000000..a109a4f --- /dev/null +++ b/pkg/s3signer/request-signature-v4_test.go @@ -0,0 +1,50 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "io" + "net/http" + "strings" + "testing" +) + +func TestRequestHost(t *testing.T) { + req, _ := buildRequest("dynamodb", "us-east-1", "{}") + req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a" + req.Host = "myhost" + canonicalHeaders := getCanonicalHeaders(*req, v4IgnoredHeaders) + + if !strings.Contains(canonicalHeaders, "host:"+req.Host) { + t.Errorf("canonical host header invalid") + } +} + +func buildRequest(serviceName, region, body string) (*http.Request, io.ReadSeeker) { + endpoint := "https://" + serviceName + "." + region + ".amazonaws.com" + reader := strings.NewReader(body) + req, _ := http.NewRequest("POST", endpoint, reader) + req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()" + req.Header.Add("X-Amz-Target", "prefix.Operation") + req.Header.Add("Content-Type", "application/x-amz-json-1.0") + req.Header.Add("Content-Length", string(len(body))) + req.Header.Add("X-Amz-Meta-Other-Header", "some-value=!@#$%^&* (+)") + req.Header.Add("X-Amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)") + req.Header.Add("X-amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)") + return req, reader +} diff --git a/pkg/s3signer/request-signature_test.go b/pkg/s3signer/request-signature_test.go new file mode 100644 index 0000000..75115d1 --- /dev/null +++ b/pkg/s3signer/request-signature_test.go @@ -0,0 +1,89 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "net/http" + "strings" + "testing" +) + +// Tests signature calculation. +func TestSignatureCalculationV4(t *testing.T) { + req, err := http.NewRequest("GET", "https://s3.amazonaws.com", nil) + if err != nil { + t.Fatal("Error:", err) + } + req = SignV4(*req, "", "", "", "us-east-1") + if req.Header.Get("Authorization") != "" { + t.Fatal("Error: anonymous credentials should not have Authorization header.") + } + + req = PreSignV4(*req, "", "", "", "us-east-1", 0) + if strings.Contains(req.URL.RawQuery, "X-Amz-Signature") { + t.Fatal("Error: anonymous credentials should not have Signature query resource.") + } + + req = SignV4(*req, "ACCESS-KEY", "SECRET-KEY", "", "us-east-1") + if req.Header.Get("Authorization") == "" { + t.Fatal("Error: normal credentials should have Authorization header.") + } + + req = PreSignV4(*req, "ACCESS-KEY", "SECRET-KEY", "", "us-east-1", 0) + if !strings.Contains(req.URL.RawQuery, "X-Amz-Signature") { + t.Fatal("Error: normal credentials should have Signature query resource.") + } +} + +func TestSignatureCalculationV2(t *testing.T) { + + var testCases = []struct { + endpointURL string + virtualHost bool + }{ + {endpointURL: "https://s3.amazonaws.com/", virtualHost: false}, + {endpointURL: "https://testbucket.s3.amazonaws.com/", virtualHost: true}, + } + + for i, testCase := range testCases { + req, err := http.NewRequest("GET", testCase.endpointURL, nil) + if err != nil { + t.Fatalf("Test %d, Error: %v", i+1, err) + } + + req = SignV2(*req, "", "", testCase.virtualHost) + if req.Header.Get("Authorization") != "" { + t.Fatalf("Test %d, Error: anonymous credentials should not have Authorization header.", i+1) + } + + req = PreSignV2(*req, "", "", 0, testCase.virtualHost) + if strings.Contains(req.URL.RawQuery, "Signature") { + t.Fatalf("Test %d, Error: anonymous credentials should not have Signature query resource.", i+1) + } + + req = SignV2(*req, "ACCESS-KEY", "SECRET-KEY", testCase.virtualHost) + if req.Header.Get("Authorization") == "" { + t.Fatalf("Test %d, Error: normal credentials should have Authorization header.", i+1) + } + + req = PreSignV2(*req, "ACCESS-KEY", "SECRET-KEY", 0, testCase.virtualHost) + if !strings.Contains(req.URL.RawQuery, "Signature") { + t.Fatalf("Test %d, Error: normal credentials should not have Signature query resource.", i+1) + } + } +} diff --git a/pkg/s3signer/test-utils_test.go b/pkg/s3signer/test-utils_test.go new file mode 100644 index 0000000..cf96d66 --- /dev/null +++ b/pkg/s3signer/test-utils_test.go @@ -0,0 +1,104 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "bufio" + "bytes" + "crypto/tls" + "io" + "io/ioutil" + "net/http" + "strings" +) + +// N B minio-go should compile on go1.5.3 onwards and httptest package is +// available only from go.1.7.x. The following function is taken from +// Go httptest package to be able to build on older versions of Go. + +// NewRequest returns a new incoming server Request, suitable +// for passing to an http.Handler for testing. +// +// The target is the RFC 7230 "request-target": it may be either a +// path or an absolute URL. If target is an absolute URL, the host name +// from the URL is used. Otherwise, "example.com" is used. +// +// The TLS field is set to a non-nil dummy value if target has scheme +// "https". +// +// The Request.Proto is always HTTP/1.1. +// +// An empty method means "GET". +// +// The provided body may be nil. If the body is of type *bytes.Reader, +// *strings.Reader, or *bytes.Buffer, the Request.ContentLength is +// set. +// +// NewRequest panics on error for ease of use in testing, where a +// panic is acceptable. +func NewRequest(method, target string, body io.Reader) *http.Request { + if method == "" { + method = "GET" + } + req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(method + " " + target + " HTTP/1.0\r\n\r\n"))) + if err != nil { + panic("invalid NewRequest arguments; " + err.Error()) + } + + // HTTP/1.0 was used above to avoid needing a Host field. Change it to 1.1 here. + req.Proto = "HTTP/1.1" + req.ProtoMinor = 1 + req.Close = false + + if body != nil { + switch v := body.(type) { + case *bytes.Buffer: + req.ContentLength = int64(v.Len()) + case *bytes.Reader: + req.ContentLength = int64(v.Len()) + case *strings.Reader: + req.ContentLength = int64(v.Len()) + default: + req.ContentLength = -1 + } + if rc, ok := body.(io.ReadCloser); ok { + req.Body = rc + } else { + req.Body = ioutil.NopCloser(body) + } + } + + // 192.0.2.0/24 is "TEST-NET" in RFC 5737 for use solely in + // documentation and example source code and should not be + // used publicly. + req.RemoteAddr = "192.0.2.1:1234" + + if req.Host == "" { + req.Host = "example.com" + } + + if strings.HasPrefix(target, "https://") { + req.TLS = &tls.ConnectionState{ + Version: tls.VersionTLS12, + HandshakeComplete: true, + ServerName: req.Host, + } + } + + return req +} diff --git a/pkg/s3signer/utils.go b/pkg/s3signer/utils.go new file mode 100644 index 0000000..33b1752 --- /dev/null +++ b/pkg/s3signer/utils.go @@ -0,0 +1,49 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "crypto/hmac" + "crypto/sha256" + "net/http" +) + +// unsignedPayload - value to be set to X-Amz-Content-Sha256 header when +const unsignedPayload = "UNSIGNED-PAYLOAD" + +// sum256 calculate sha256 sum for an input byte array. +func sum256(data []byte) []byte { + hash := sha256.New() + hash.Write(data) + return hash.Sum(nil) +} + +// sumHMAC calculate hmac between two input byte array. +func sumHMAC(key []byte, data []byte) []byte { + hash := hmac.New(sha256.New, key) + hash.Write(data) + return hash.Sum(nil) +} + +// getHostAddr returns host header if available, otherwise returns host from URL +func getHostAddr(req *http.Request) string { + if req.Host != "" { + return req.Host + } + return req.URL.Host +} diff --git a/pkg/s3signer/utils_test.go b/pkg/s3signer/utils_test.go new file mode 100644 index 0000000..e7fc7b3 --- /dev/null +++ b/pkg/s3signer/utils_test.go @@ -0,0 +1,86 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "fmt" + "net/http" + "net/url" + "testing" +) + +// Tests url encoding. +func TestEncodeURL2Path(t *testing.T) { + type urlStrings struct { + virtualHost bool + bucketName string + objName string + encodedObjName string + } + + want := []urlStrings{ + { + virtualHost: true, + bucketName: "bucketName", + objName: "本語", + encodedObjName: "%E6%9C%AC%E8%AA%9E", + }, + { + virtualHost: true, + bucketName: "bucketName", + objName: "本語.1", + encodedObjName: "%E6%9C%AC%E8%AA%9E.1", + }, + { + virtualHost: true, + objName: ">123>3123123", + bucketName: "bucketName", + encodedObjName: "%3E123%3E3123123", + }, + { + virtualHost: true, + bucketName: "bucketName", + objName: "test 1 2.txt", + encodedObjName: "test%201%202.txt", + }, + { + virtualHost: false, + bucketName: "test.bucketName", + objName: "test++ 1.txt", + encodedObjName: "test%2B%2B%201.txt", + }, + } + + for i, o := range want { + var hostURL string + if o.virtualHost { + hostURL = fmt.Sprintf("https://%s.s3.amazonaws.com/%s", o.bucketName, o.objName) + } else { + hostURL = fmt.Sprintf("https://s3.amazonaws.com/%s/%s", o.bucketName, o.objName) + } + u, err := url.Parse(hostURL) + if err != nil { + t.Fatalf("Test %d, Error: %v", i+1, err) + } + expectedPath := "/" + o.bucketName + "/" + o.encodedObjName + if foundPath := encodeURL2Path(&http.Request{URL: u}, o.virtualHost); foundPath != expectedPath { + t.Fatalf("Test %d, Error: expected = `%v`, found = `%v`", i+1, expectedPath, foundPath) + } + } + +} diff --git a/pkg/s3utils/utils.go b/pkg/s3utils/utils.go new file mode 100644 index 0000000..adceb7f --- /dev/null +++ b/pkg/s3utils/utils.go @@ -0,0 +1,331 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3utils + +import ( + "bytes" + "encoding/hex" + "errors" + "net" + "net/url" + "regexp" + "sort" + "strings" + "unicode/utf8" +) + +// Sentinel URL is the default url value which is invalid. +var sentinelURL = url.URL{} + +// IsValidDomain validates if input string is a valid domain name. +func IsValidDomain(host string) bool { + // See RFC 1035, RFC 3696. + host = strings.TrimSpace(host) + if len(host) == 0 || len(host) > 255 { + return false + } + // host cannot start or end with "-" + if host[len(host)-1:] == "-" || host[:1] == "-" { + return false + } + // host cannot start or end with "_" + if host[len(host)-1:] == "_" || host[:1] == "_" { + return false + } + // host cannot start or end with a "." + if host[len(host)-1:] == "." || host[:1] == "." { + return false + } + // All non alphanumeric characters are invalid. + if strings.ContainsAny(host, "`~!@#$%^&*()+={}[]|\\\"';:> 1 { + return parts[1] + } + parts = amazonS3HostHyphen.FindStringSubmatch(endpointURL.Host) + if len(parts) > 1 { + return parts[1] + } + parts = amazonS3ChinaHost.FindStringSubmatch(endpointURL.Host) + if len(parts) > 1 { + return parts[1] + } + parts = amazonS3HostDot.FindStringSubmatch(endpointURL.Host) + if len(parts) > 1 { + return parts[1] + } + return "" +} + +// IsAmazonEndpoint - Match if it is exactly Amazon S3 endpoint. +func IsAmazonEndpoint(endpointURL url.URL) bool { + if endpointURL.Host == "s3-external-1.amazonaws.com" || endpointURL.Host == "s3.amazonaws.com" { + return true + } + return GetRegionFromURL(endpointURL) != "" +} + +// IsAmazonGovCloudEndpoint - Match if it is exactly Amazon S3 GovCloud endpoint. +func IsAmazonGovCloudEndpoint(endpointURL url.URL) bool { + if endpointURL == sentinelURL { + return false + } + return (endpointURL.Host == "s3-us-gov-west-1.amazonaws.com" || + IsAmazonFIPSGovCloudEndpoint(endpointURL)) +} + +// IsAmazonFIPSGovCloudEndpoint - Match if it is exactly Amazon S3 FIPS GovCloud endpoint. +// See https://aws.amazon.com/compliance/fips. +func IsAmazonFIPSGovCloudEndpoint(endpointURL url.URL) bool { + if endpointURL == sentinelURL { + return false + } + return endpointURL.Host == "s3-fips-us-gov-west-1.amazonaws.com" || + endpointURL.Host == "s3-fips.dualstack.us-gov-west-1.amazonaws.com" +} + +// IsAmazonFIPSUSEastWestEndpoint - Match if it is exactly Amazon S3 FIPS US East/West endpoint. +// See https://aws.amazon.com/compliance/fips. +func IsAmazonFIPSUSEastWestEndpoint(endpointURL url.URL) bool { + if endpointURL == sentinelURL { + return false + } + switch endpointURL.Host { + case "s3-fips.us-east-2.amazonaws.com": + case "s3-fips.dualstack.us-west-1.amazonaws.com": + case "s3-fips.dualstack.us-west-2.amazonaws.com": + case "s3-fips.dualstack.us-east-2.amazonaws.com": + case "s3-fips.dualstack.us-east-1.amazonaws.com": + case "s3-fips.us-west-1.amazonaws.com": + case "s3-fips.us-west-2.amazonaws.com": + case "s3-fips.us-east-1.amazonaws.com": + default: + return false + } + return true +} + +// IsAmazonFIPSEndpoint - Match if it is exactly Amazon S3 FIPS endpoint. +// See https://aws.amazon.com/compliance/fips. +func IsAmazonFIPSEndpoint(endpointURL url.URL) bool { + return IsAmazonFIPSUSEastWestEndpoint(endpointURL) || IsAmazonFIPSGovCloudEndpoint(endpointURL) +} + +// IsGoogleEndpoint - Match if it is exactly Google cloud storage endpoint. +func IsGoogleEndpoint(endpointURL url.URL) bool { + if endpointURL == sentinelURL { + return false + } + return endpointURL.Host == "storage.googleapis.com" +} + +// Expects ascii encoded strings - from output of urlEncodePath +func percentEncodeSlash(s string) string { + return strings.Replace(s, "/", "%2F", -1) +} + +// QueryEncode - encodes query values in their URL encoded form. In +// addition to the percent encoding performed by urlEncodePath() used +// here, it also percent encodes '/' (forward slash) +func QueryEncode(v url.Values) string { + if v == nil { + return "" + } + var buf bytes.Buffer + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + vs := v[k] + prefix := percentEncodeSlash(EncodePath(k)) + "=" + for _, v := range vs { + if buf.Len() > 0 { + buf.WriteByte('&') + } + buf.WriteString(prefix) + buf.WriteString(percentEncodeSlash(EncodePath(v))) + } + } + return buf.String() +} + +// if object matches reserved string, no need to encode them +var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") + +// EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences +// +// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 +// non english characters cannot be parsed due to the nature in which url.Encode() is written +// +// This function on the other hand is a direct replacement for url.Encode() technique to support +// pretty much every UTF-8 character. +func EncodePath(pathName string) string { + if reservedObjectNames.MatchString(pathName) { + return pathName + } + var encodedPathname string + for _, s := range pathName { + if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) + encodedPathname = encodedPathname + string(s) + continue + } + switch s { + case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) + encodedPathname = encodedPathname + string(s) + continue + default: + len := utf8.RuneLen(s) + if len < 0 { + // if utf8 cannot convert return the same string as is + return pathName + } + u := make([]byte, len) + utf8.EncodeRune(u, s) + for _, r := range u { + hex := hex.EncodeToString([]byte{r}) + encodedPathname = encodedPathname + "%" + strings.ToUpper(hex) + } + } + } + return encodedPathname +} + +// We support '.' with bucket names but we fallback to using path +// style requests instead for such buckets. +var ( + validBucketName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9\.\-\_\:]{1,61}[A-Za-z0-9]$`) + validBucketNameStrict = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`) + ipAddress = regexp.MustCompile(`^(\d+\.){3}\d+$`) +) + +// Common checker for both stricter and basic validation. +func checkBucketNameCommon(bucketName string, strict bool) (err error) { + if strings.TrimSpace(bucketName) == "" { + return errors.New("Bucket name cannot be empty") + } + if len(bucketName) < 3 { + return errors.New("Bucket name cannot be smaller than 3 characters") + } + if len(bucketName) > 63 { + return errors.New("Bucket name cannot be greater than 63 characters") + } + if ipAddress.MatchString(bucketName) { + return errors.New("Bucket name cannot be an ip address") + } + if strings.Contains(bucketName, "..") { + return errors.New("Bucket name contains invalid characters") + } + if strict { + if !validBucketNameStrict.MatchString(bucketName) { + err = errors.New("Bucket name contains invalid characters") + } + return err + } + if !validBucketName.MatchString(bucketName) { + err = errors.New("Bucket name contains invalid characters") + } + return err +} + +// CheckValidBucketName - checks if we have a valid input bucket name. +func CheckValidBucketName(bucketName string) (err error) { + return checkBucketNameCommon(bucketName, false) +} + +// CheckValidBucketNameStrict - checks if we have a valid input bucket name. +// This is a stricter version. +// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html +func CheckValidBucketNameStrict(bucketName string) (err error) { + return checkBucketNameCommon(bucketName, true) +} + +// CheckValidObjectNamePrefix - checks if we have a valid input object name prefix. +// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html +func CheckValidObjectNamePrefix(objectName string) error { + if len(objectName) > 1024 { + return errors.New("Object name cannot be greater than 1024 characters") + } + if !utf8.ValidString(objectName) { + return errors.New("Object name with non UTF-8 strings are not supported") + } + return nil +} + +// CheckValidObjectName - checks if we have a valid input object name. +// - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html +func CheckValidObjectName(objectName string) error { + if strings.TrimSpace(objectName) == "" { + return errors.New("Object name cannot be empty") + } + return CheckValidObjectNamePrefix(objectName) +} diff --git a/pkg/s3utils/utils_test.go b/pkg/s3utils/utils_test.go new file mode 100644 index 0000000..55eaaea --- /dev/null +++ b/pkg/s3utils/utils_test.go @@ -0,0 +1,408 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3utils + +import ( + "errors" + "net/url" + "testing" +) + +// Tests get region from host URL. +func TestGetRegionFromURL(t *testing.T) { + testCases := []struct { + u url.URL + expectedRegion string + }{ + { + u: url.URL{Host: "storage.googleapis.com"}, + expectedRegion: "", + }, + { + u: url.URL{Host: "s3.cn-north-1.amazonaws.com.cn"}, + expectedRegion: "cn-north-1", + }, + { + u: url.URL{Host: "s3.cn-northwest-1.amazonaws.com.cn"}, + expectedRegion: "cn-northwest-1", + }, + { + u: url.URL{Host: "s3-fips-us-gov-west-1.amazonaws.com"}, + expectedRegion: "us-gov-west-1", + }, + { + u: url.URL{Host: "s3-us-gov-west-1.amazonaws.com"}, + expectedRegion: "us-gov-west-1", + }, + { + u: url.URL{Host: "192.168.1.1"}, + expectedRegion: "", + }, + { + u: url.URL{Host: "s3-eu-west-1.amazonaws.com"}, + expectedRegion: "eu-west-1", + }, + { + u: url.URL{Host: "s3.eu-west-1.amazonaws.com"}, + expectedRegion: "eu-west-1", + }, + { + u: url.URL{Host: "s3.dualstack.eu-west-1.amazonaws.com"}, + expectedRegion: "eu-west-1", + }, + { + u: url.URL{Host: "s3.amazonaws.com"}, + expectedRegion: "", + }, + { + u: url.URL{Host: "s3-external-1.amazonaws.com"}, + expectedRegion: "", + }, + } + + for i, testCase := range testCases { + region := GetRegionFromURL(testCase.u) + if testCase.expectedRegion != region { + t.Errorf("Test %d: Expected region %s, got %s", i+1, testCase.expectedRegion, region) + } + } +} + +// Tests for 'isValidDomain(host string) bool'. +func TestIsValidDomain(t *testing.T) { + testCases := []struct { + // Input. + host string + // Expected result. + result bool + }{ + {"s3.amazonaws.com", true}, + {"s3.cn-north-1.amazonaws.com.cn", true}, + {"s3.cn-northwest-1.amazonaws.com.cn", true}, + {"s3.amazonaws.com_", false}, + {"%$$$", false}, + {"s3.amz.test.com", true}, + {"s3.%%", false}, + {"localhost", true}, + {"-localhost", false}, + {"", false}, + {"\n \t", false}, + {" ", false}, + } + + for i, testCase := range testCases { + result := IsValidDomain(testCase.host) + if testCase.result != result { + t.Errorf("Test %d: Expected isValidDomain test to be '%v', but found '%v' instead", i+1, testCase.result, result) + } + } +} + +// Tests validate IP address validator. +func TestIsValidIP(t *testing.T) { + testCases := []struct { + // Input. + ip string + // Expected result. + result bool + }{ + {"192.168.1.1", true}, + {"192.168.1", false}, + {"192.168.1.1.1", false}, + {"-192.168.1.1", false}, + {"260.192.1.1", false}, + } + + for i, testCase := range testCases { + result := IsValidIP(testCase.ip) + if testCase.result != result { + t.Errorf("Test %d: Expected isValidIP to be '%v' for input \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.ip, result) + } + } + +} + +// Tests validate virtual host validator. +func TestIsVirtualHostSupported(t *testing.T) { + testCases := []struct { + url string + bucket string + // Expeceted result. + result bool + }{ + {"https://s3.amazonaws.com", "my-bucket", true}, + {"https://s3.cn-north-1.amazonaws.com.cn", "my-bucket", true}, + {"https://s3.amazonaws.com", "my-bucket.", false}, + {"https://amazons3.amazonaws.com", "my-bucket.", false}, + {"https://storage.googleapis.com/", "my-bucket", true}, + {"https://mystorage.googleapis.com/", "my-bucket", false}, + } + + for i, testCase := range testCases { + u, err := url.Parse(testCase.url) + if err != nil { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err) + } + result := IsVirtualHostSupported(*u, testCase.bucket) + if testCase.result != result { + t.Errorf("Test %d: Expected isVirtualHostSupported to be '%v' for input url \"%s\" and bucket \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.url, testCase.bucket, result) + } + } +} + +// Tests validate Amazon endpoint validator. +func TestIsAmazonEndpoint(t *testing.T) { + testCases := []struct { + url string + // Expected result. + result bool + }{ + {"https://192.168.1.1", false}, + {"192.168.1.1", false}, + {"http://storage.googleapis.com", false}, + {"https://storage.googleapis.com", false}, + {"storage.googleapis.com", false}, + {"s3.amazonaws.com", false}, + {"https://amazons3.amazonaws.com", false}, + {"-192.168.1.1", false}, + {"260.192.1.1", false}, + {"https://s3-.amazonaws.com", false}, + {"https://s3..amazonaws.com", false}, + {"https://s3.dualstack.us-west-1.amazonaws.com.cn", false}, + {"https://s3..us-west-1.amazonaws.com.cn", false}, + // valid inputs. + {"https://s3.amazonaws.com", true}, + {"https://s3-external-1.amazonaws.com", true}, + {"https://s3.cn-north-1.amazonaws.com.cn", true}, + {"https://s3-us-west-1.amazonaws.com", true}, + {"https://s3.us-west-1.amazonaws.com", true}, + {"https://s3.dualstack.us-west-1.amazonaws.com", true}, + } + + for i, testCase := range testCases { + u, err := url.Parse(testCase.url) + if err != nil { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err) + } + result := IsAmazonEndpoint(*u) + if testCase.result != result { + t.Errorf("Test %d: Expected isAmazonEndpoint to be '%v' for input \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.url, result) + } + } + +} + +// Tests validate Google Cloud end point validator. +func TestIsGoogleEndpoint(t *testing.T) { + testCases := []struct { + url string + // Expected result. + result bool + }{ + {"192.168.1.1", false}, + {"https://192.168.1.1", false}, + {"s3.amazonaws.com", false}, + {"http://s3.amazonaws.com", false}, + {"https://s3.amazonaws.com", false}, + {"https://s3.cn-north-1.amazonaws.com.cn", false}, + {"-192.168.1.1", false}, + {"260.192.1.1", false}, + // valid inputs. + {"http://storage.googleapis.com", true}, + {"https://storage.googleapis.com", true}, + } + + for i, testCase := range testCases { + u, err := url.Parse(testCase.url) + if err != nil { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err) + } + result := IsGoogleEndpoint(*u) + if testCase.result != result { + t.Errorf("Test %d: Expected isGoogleEndpoint to be '%v' for input \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.url, result) + } + } + +} + +func TestPercentEncodeSlash(t *testing.T) { + testCases := []struct { + input string + output string + }{ + {"test123", "test123"}, + {"abc,+_1", "abc,+_1"}, + {"%40prefix=test%40123", "%40prefix=test%40123"}, + {"key1=val1/val2", "key1=val1%2Fval2"}, + {"%40prefix=test%40123/", "%40prefix=test%40123%2F"}, + } + + for i, testCase := range testCases { + receivedOutput := percentEncodeSlash(testCase.input) + if testCase.output != receivedOutput { + t.Errorf( + "Test %d: Input: \"%s\" --> Expected percentEncodeSlash to return \"%s\", but it returned \"%s\" instead!", + i+1, testCase.input, testCase.output, + receivedOutput, + ) + + } + } +} + +// Tests validate the query encoder. +func TestQueryEncode(t *testing.T) { + testCases := []struct { + queryKey string + valueToEncode []string + // Expected result. + result string + }{ + {"prefix", []string{"test@123", "test@456"}, "prefix=test%40123&prefix=test%40456"}, + {"@prefix", []string{"test@123"}, "%40prefix=test%40123"}, + {"@prefix", []string{"a/b/c/"}, "%40prefix=a%2Fb%2Fc%2F"}, + {"prefix", []string{"test#123"}, "prefix=test%23123"}, + {"prefix#", []string{"test#123"}, "prefix%23=test%23123"}, + {"prefix", []string{"test123"}, "prefix=test123"}, + {"prefix", []string{"test本語123", "test123"}, "prefix=test%E6%9C%AC%E8%AA%9E123&prefix=test123"}, + } + + for i, testCase := range testCases { + urlValues := make(url.Values) + for _, valueToEncode := range testCase.valueToEncode { + urlValues.Add(testCase.queryKey, valueToEncode) + } + result := QueryEncode(urlValues) + if testCase.result != result { + t.Errorf("Test %d: Expected queryEncode result to be \"%s\", but found it to be \"%s\" instead", i+1, testCase.result, result) + } + } +} + +// Tests validate the URL path encoder. +func TestEncodePath(t *testing.T) { + testCases := []struct { + // Input. + inputStr string + // Expected result. + result string + }{ + {"thisisthe%url", "thisisthe%25url"}, + {"本語", "%E6%9C%AC%E8%AA%9E"}, + {"本語.1", "%E6%9C%AC%E8%AA%9E.1"}, + {">123", "%3E123"}, + {"myurl#link", "myurl%23link"}, + {"space in url", "space%20in%20url"}, + {"url+path", "url%2Bpath"}, + } + + for i, testCase := range testCases { + result := EncodePath(testCase.inputStr) + if testCase.result != result { + t.Errorf("Test %d: Expected queryEncode result to be \"%s\", but found it to be \"%s\" instead", i+1, testCase.result, result) + } + } +} + +// Tests validate the bucket name validator. +func TestIsValidBucketName(t *testing.T) { + testCases := []struct { + // Input. + bucketName string + // Expected result. + err error + // Flag to indicate whether test should Pass. + shouldPass bool + }{ + {".mybucket", errors.New("Bucket name contains invalid characters"), false}, + {"$mybucket", errors.New("Bucket name contains invalid characters"), false}, + {"mybucket-", errors.New("Bucket name contains invalid characters"), false}, + {"my", errors.New("Bucket name cannot be smaller than 3 characters"), false}, + {"", errors.New("Bucket name cannot be empty"), false}, + {"my..bucket", errors.New("Bucket name contains invalid characters"), false}, + {"192.168.1.168", errors.New("Bucket name cannot be an ip address"), false}, + {":bucketname", errors.New("Bucket name contains invalid characters"), false}, + {"_bucketName", errors.New("Bucket name contains invalid characters"), false}, + {"my.bucket.com", nil, true}, + {"my-bucket", nil, true}, + {"123my-bucket", nil, true}, + {"Mybucket", nil, true}, + {"My_bucket", nil, true}, + {"My:bucket", nil, true}, + } + + for i, testCase := range testCases { + err := CheckValidBucketName(testCase.bucketName) + if err != nil && testCase.shouldPass { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) + } + if err == nil && !testCase.shouldPass { + t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if err.Error() != testCase.err.Error() { + t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) + } + } + + } + +} + +// Tests validate the bucket name validator stricter. +func TestIsValidBucketNameStrict(t *testing.T) { + testCases := []struct { + // Input. + bucketName string + // Expected result. + err error + // Flag to indicate whether test should Pass. + shouldPass bool + }{ + {".mybucket", errors.New("Bucket name contains invalid characters"), false}, + {"$mybucket", errors.New("Bucket name contains invalid characters"), false}, + {"mybucket-", errors.New("Bucket name contains invalid characters"), false}, + {"my", errors.New("Bucket name cannot be smaller than 3 characters"), false}, + {"", errors.New("Bucket name cannot be empty"), false}, + {"my..bucket", errors.New("Bucket name contains invalid characters"), false}, + {"192.168.1.168", errors.New("Bucket name cannot be an ip address"), false}, + {"Mybucket", errors.New("Bucket name contains invalid characters"), false}, + {"my.bucket.com", nil, true}, + {"my-bucket", nil, true}, + {"123my-bucket", nil, true}, + } + + for i, testCase := range testCases { + err := CheckValidBucketNameStrict(testCase.bucketName) + if err != nil && testCase.shouldPass { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) + } + if err == nil && !testCase.shouldPass { + t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if err.Error() != testCase.err.Error() { + t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) + } + } + + } + +} diff --git a/pkg/set/stringset.go b/pkg/set/stringset.go new file mode 100644 index 0000000..efd0262 --- /dev/null +++ b/pkg/set/stringset.go @@ -0,0 +1,197 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package set + +import ( + "encoding/json" + "fmt" + "sort" +) + +// StringSet - uses map as set of strings. +type StringSet map[string]struct{} + +// ToSlice - returns StringSet as string slice. +func (set StringSet) ToSlice() []string { + keys := make([]string, 0, len(set)) + for k := range set { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// IsEmpty - returns whether the set is empty or not. +func (set StringSet) IsEmpty() bool { + return len(set) == 0 +} + +// Add - adds string to the set. +func (set StringSet) Add(s string) { + set[s] = struct{}{} +} + +// Remove - removes string in the set. It does nothing if string does not exist in the set. +func (set StringSet) Remove(s string) { + delete(set, s) +} + +// Contains - checks if string is in the set. +func (set StringSet) Contains(s string) bool { + _, ok := set[s] + return ok +} + +// FuncMatch - returns new set containing each value who passes match function. +// A 'matchFn' should accept element in a set as first argument and +// 'matchString' as second argument. The function can do any logic to +// compare both the arguments and should return true to accept element in +// a set to include in output set else the element is ignored. +func (set StringSet) FuncMatch(matchFn func(string, string) bool, matchString string) StringSet { + nset := NewStringSet() + for k := range set { + if matchFn(k, matchString) { + nset.Add(k) + } + } + return nset +} + +// ApplyFunc - returns new set containing each value processed by 'applyFn'. +// A 'applyFn' should accept element in a set as a argument and return +// a processed string. The function can do any logic to return a processed +// string. +func (set StringSet) ApplyFunc(applyFn func(string) string) StringSet { + nset := NewStringSet() + for k := range set { + nset.Add(applyFn(k)) + } + return nset +} + +// Equals - checks whether given set is equal to current set or not. +func (set StringSet) Equals(sset StringSet) bool { + // If length of set is not equal to length of given set, the + // set is not equal to given set. + if len(set) != len(sset) { + return false + } + + // As both sets are equal in length, check each elements are equal. + for k := range set { + if _, ok := sset[k]; !ok { + return false + } + } + + return true +} + +// Intersection - returns the intersection with given set as new set. +func (set StringSet) Intersection(sset StringSet) StringSet { + nset := NewStringSet() + for k := range set { + if _, ok := sset[k]; ok { + nset.Add(k) + } + } + + return nset +} + +// Difference - returns the difference with given set as new set. +func (set StringSet) Difference(sset StringSet) StringSet { + nset := NewStringSet() + for k := range set { + if _, ok := sset[k]; !ok { + nset.Add(k) + } + } + + return nset +} + +// Union - returns the union with given set as new set. +func (set StringSet) Union(sset StringSet) StringSet { + nset := NewStringSet() + for k := range set { + nset.Add(k) + } + + for k := range sset { + nset.Add(k) + } + + return nset +} + +// MarshalJSON - converts to JSON data. +func (set StringSet) MarshalJSON() ([]byte, error) { + return json.Marshal(set.ToSlice()) +} + +// UnmarshalJSON - parses JSON data and creates new set with it. +// If 'data' contains JSON string array, the set contains each string. +// If 'data' contains JSON string, the set contains the string as one element. +// If 'data' contains Other JSON types, JSON parse error is returned. +func (set *StringSet) UnmarshalJSON(data []byte) error { + sl := []string{} + var err error + if err = json.Unmarshal(data, &sl); err == nil { + *set = make(StringSet) + for _, s := range sl { + set.Add(s) + } + } else { + var s string + if err = json.Unmarshal(data, &s); err == nil { + *set = make(StringSet) + set.Add(s) + } + } + + return err +} + +// String - returns printable string of the set. +func (set StringSet) String() string { + return fmt.Sprintf("%s", set.ToSlice()) +} + +// NewStringSet - creates new string set. +func NewStringSet() StringSet { + return make(StringSet) +} + +// CreateStringSet - creates new string set with given string values. +func CreateStringSet(sl ...string) StringSet { + set := make(StringSet) + for _, k := range sl { + set.Add(k) + } + return set +} + +// CopyStringSet - returns copy of given set. +func CopyStringSet(set StringSet) StringSet { + nset := NewStringSet() + for k, v := range set { + nset[k] = v + } + return nset +} diff --git a/pkg/set/stringset_test.go b/pkg/set/stringset_test.go new file mode 100644 index 0000000..d7e6aa7 --- /dev/null +++ b/pkg/set/stringset_test.go @@ -0,0 +1,348 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package set + +import ( + "fmt" + "strings" + "testing" +) + +// NewStringSet() is called and the result is validated. +func TestNewStringSet(t *testing.T) { + if ss := NewStringSet(); !ss.IsEmpty() { + t.Fatalf("expected: true, got: false") + } +} + +// CreateStringSet() is called and the result is validated. +func TestCreateStringSet(t *testing.T) { + ss := CreateStringSet("foo") + if str := ss.String(); str != `[foo]` { + t.Fatalf("expected: %s, got: %s", `["foo"]`, str) + } +} + +// CopyStringSet() is called and the result is validated. +func TestCopyStringSet(t *testing.T) { + ss := CreateStringSet("foo") + sscopy := CopyStringSet(ss) + if !ss.Equals(sscopy) { + t.Fatalf("expected: %s, got: %s", ss, sscopy) + } +} + +// StringSet.Add() is called with series of cases for valid and erroneous inputs and the result is validated. +func TestStringSetAdd(t *testing.T) { + testCases := []struct { + value string + expectedResult string + }{ + // Test first addition. + {"foo", `[foo]`}, + // Test duplicate addition. + {"foo", `[foo]`}, + // Test new addition. + {"bar", `[bar foo]`}, + } + + ss := NewStringSet() + for _, testCase := range testCases { + ss.Add(testCase.value) + if str := ss.String(); str != testCase.expectedResult { + t.Fatalf("expected: %s, got: %s", testCase.expectedResult, str) + } + } +} + +// StringSet.Remove() is called with series of cases for valid and erroneous inputs and the result is validated. +func TestStringSetRemove(t *testing.T) { + ss := CreateStringSet("foo", "bar") + testCases := []struct { + value string + expectedResult string + }{ + // Test removing non-existen item. + {"baz", `[bar foo]`}, + // Test remove existing item. + {"foo", `[bar]`}, + // Test remove existing item again. + {"foo", `[bar]`}, + // Test remove to make set to empty. + {"bar", `[]`}, + } + + for _, testCase := range testCases { + ss.Remove(testCase.value) + if str := ss.String(); str != testCase.expectedResult { + t.Fatalf("expected: %s, got: %s", testCase.expectedResult, str) + } + } +} + +// StringSet.Contains() is called with series of cases for valid and erroneous inputs and the result is validated. +func TestStringSetContains(t *testing.T) { + ss := CreateStringSet("foo") + testCases := []struct { + value string + expectedResult bool + }{ + // Test to check non-existent item. + {"bar", false}, + // Test to check existent item. + {"foo", true}, + // Test to verify case sensitivity. + {"Foo", false}, + } + + for _, testCase := range testCases { + if result := ss.Contains(testCase.value); result != testCase.expectedResult { + t.Fatalf("expected: %t, got: %t", testCase.expectedResult, result) + } + } +} + +// StringSet.FuncMatch() is called with series of cases for valid and erroneous inputs and the result is validated. +func TestStringSetFuncMatch(t *testing.T) { + ss := CreateStringSet("foo", "bar") + testCases := []struct { + matchFn func(string, string) bool + value string + expectedResult string + }{ + // Test to check match function doing case insensive compare. + {func(setValue string, compareValue string) bool { + return strings.ToUpper(setValue) == strings.ToUpper(compareValue) + }, "Bar", `[bar]`}, + // Test to check match function doing prefix check. + {func(setValue string, compareValue string) bool { + return strings.HasPrefix(compareValue, setValue) + }, "foobar", `[foo]`}, + } + + for _, testCase := range testCases { + s := ss.FuncMatch(testCase.matchFn, testCase.value) + if result := s.String(); result != testCase.expectedResult { + t.Fatalf("expected: %s, got: %s", testCase.expectedResult, result) + } + } +} + +// StringSet.ApplyFunc() is called with series of cases for valid and erroneous inputs and the result is validated. +func TestStringSetApplyFunc(t *testing.T) { + ss := CreateStringSet("foo", "bar") + testCases := []struct { + applyFn func(string) string + expectedResult string + }{ + // Test to apply function prepending a known string. + {func(setValue string) string { return "mybucket/" + setValue }, `[mybucket/bar mybucket/foo]`}, + // Test to apply function modifying values. + {func(setValue string) string { return setValue[1:] }, `[ar oo]`}, + } + + for _, testCase := range testCases { + s := ss.ApplyFunc(testCase.applyFn) + if result := s.String(); result != testCase.expectedResult { + t.Fatalf("expected: %s, got: %s", testCase.expectedResult, result) + } + } +} + +// StringSet.Equals() is called with series of cases for valid and erroneous inputs and the result is validated. +func TestStringSetEquals(t *testing.T) { + testCases := []struct { + set1 StringSet + set2 StringSet + expectedResult bool + }{ + // Test equal set + {CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar"), true}, + // Test second set with more items + {CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar", "baz"), false}, + // Test second set with less items + {CreateStringSet("foo", "bar"), CreateStringSet("bar"), false}, + } + + for _, testCase := range testCases { + if result := testCase.set1.Equals(testCase.set2); result != testCase.expectedResult { + t.Fatalf("expected: %t, got: %t", testCase.expectedResult, result) + } + } +} + +// StringSet.Intersection() is called with series of cases for valid and erroneous inputs and the result is validated. +func TestStringSetIntersection(t *testing.T) { + testCases := []struct { + set1 StringSet + set2 StringSet + expectedResult StringSet + }{ + // Test intersecting all values. + {CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar")}, + // Test intersecting all values in second set. + {CreateStringSet("foo", "bar", "baz"), CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar")}, + // Test intersecting different values in second set. + {CreateStringSet("foo", "baz"), CreateStringSet("baz", "bar"), CreateStringSet("baz")}, + // Test intersecting none. + {CreateStringSet("foo", "baz"), CreateStringSet("poo", "bar"), NewStringSet()}, + } + + for _, testCase := range testCases { + if result := testCase.set1.Intersection(testCase.set2); !result.Equals(testCase.expectedResult) { + t.Fatalf("expected: %s, got: %s", testCase.expectedResult, result) + } + } +} + +// StringSet.Difference() is called with series of cases for valid and erroneous inputs and the result is validated. +func TestStringSetDifference(t *testing.T) { + testCases := []struct { + set1 StringSet + set2 StringSet + expectedResult StringSet + }{ + // Test differing none. + {CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar"), NewStringSet()}, + // Test differing in first set. + {CreateStringSet("foo", "bar", "baz"), CreateStringSet("foo", "bar"), CreateStringSet("baz")}, + // Test differing values in both set. + {CreateStringSet("foo", "baz"), CreateStringSet("baz", "bar"), CreateStringSet("foo")}, + // Test differing all values. + {CreateStringSet("foo", "baz"), CreateStringSet("poo", "bar"), CreateStringSet("foo", "baz")}, + } + + for _, testCase := range testCases { + if result := testCase.set1.Difference(testCase.set2); !result.Equals(testCase.expectedResult) { + t.Fatalf("expected: %s, got: %s", testCase.expectedResult, result) + } + } +} + +// StringSet.Union() is called with series of cases for valid and erroneous inputs and the result is validated. +func TestStringSetUnion(t *testing.T) { + testCases := []struct { + set1 StringSet + set2 StringSet + expectedResult StringSet + }{ + // Test union same values. + {CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar")}, + // Test union same values in second set. + {CreateStringSet("foo", "bar", "baz"), CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar", "baz")}, + // Test union different values in both set. + {CreateStringSet("foo", "baz"), CreateStringSet("baz", "bar"), CreateStringSet("foo", "baz", "bar")}, + // Test union all different values. + {CreateStringSet("foo", "baz"), CreateStringSet("poo", "bar"), CreateStringSet("foo", "baz", "poo", "bar")}, + } + + for _, testCase := range testCases { + if result := testCase.set1.Union(testCase.set2); !result.Equals(testCase.expectedResult) { + t.Fatalf("expected: %s, got: %s", testCase.expectedResult, result) + } + } +} + +// StringSet.MarshalJSON() is called with series of cases for valid and erroneous inputs and the result is validated. +func TestStringSetMarshalJSON(t *testing.T) { + testCases := []struct { + set StringSet + expectedResult string + }{ + // Test set with values. + {CreateStringSet("foo", "bar"), `["bar","foo"]`}, + // Test empty set. + {NewStringSet(), "[]"}, + } + + for _, testCase := range testCases { + if result, _ := testCase.set.MarshalJSON(); string(result) != testCase.expectedResult { + t.Fatalf("expected: %s, got: %s", testCase.expectedResult, string(result)) + } + } +} + +// StringSet.UnmarshalJSON() is called with series of cases for valid and erroneous inputs and the result is validated. +func TestStringSetUnmarshalJSON(t *testing.T) { + testCases := []struct { + data []byte + expectedResult string + }{ + // Test to convert JSON array to set. + {[]byte(`["bar","foo"]`), `[bar foo]`}, + // Test to convert JSON string to set. + {[]byte(`"bar"`), `[bar]`}, + // Test to convert JSON empty array to set. + {[]byte(`[]`), `[]`}, + // Test to convert JSON empty string to set. + {[]byte(`""`), `[]`}, + } + + for _, testCase := range testCases { + var set StringSet + set.UnmarshalJSON(testCase.data) + if result := set.String(); result != testCase.expectedResult { + t.Fatalf("expected: %s, got: %s", testCase.expectedResult, result) + } + } +} + +// StringSet.String() is called with series of cases for valid and erroneous inputs and the result is validated. +func TestStringSetString(t *testing.T) { + testCases := []struct { + set StringSet + expectedResult string + }{ + // Test empty set. + {NewStringSet(), `[]`}, + // Test set with empty value. + {CreateStringSet(""), `[]`}, + // Test set with value. + {CreateStringSet("foo"), `[foo]`}, + } + + for _, testCase := range testCases { + if str := testCase.set.String(); str != testCase.expectedResult { + t.Fatalf("expected: %s, got: %s", testCase.expectedResult, str) + } + } +} + +// StringSet.ToSlice() is called with series of cases for valid and erroneous inputs and the result is validated. +func TestStringSetToSlice(t *testing.T) { + testCases := []struct { + set StringSet + expectedResult string + }{ + // Test empty set. + {NewStringSet(), `[]`}, + // Test set with empty value. + {CreateStringSet(""), `[]`}, + // Test set with value. + {CreateStringSet("foo"), `[foo]`}, + // Test set with value. + {CreateStringSet("foo", "bar"), `[bar foo]`}, + } + + for _, testCase := range testCases { + sslice := testCase.set.ToSlice() + if str := fmt.Sprintf("%s", sslice); str != testCase.expectedResult { + t.Fatalf("expected: %s, got: %s", testCase.expectedResult, str) + } + } +} diff --git a/post-policy.go b/post-policy.go new file mode 100644 index 0000000..c285fde --- /dev/null +++ b/post-policy.go @@ -0,0 +1,270 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "encoding/base64" + "fmt" + "strings" + "time" +) + +// expirationDateFormat date format for expiration key in json policy. +const expirationDateFormat = "2006-01-02T15:04:05.999Z" + +// policyCondition explanation: +// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html +// +// Example: +// +// policyCondition { +// matchType: "$eq", +// key: "$Content-Type", +// value: "image/png", +// } +// +type policyCondition struct { + matchType string + condition string + value string +} + +// PostPolicy - Provides strict static type conversion and validation +// for Amazon S3's POST policy JSON string. +type PostPolicy struct { + // Expiration date and time of the POST policy. + expiration time.Time + // Collection of different policy conditions. + conditions []policyCondition + // ContentLengthRange minimum and maximum allowable size for the + // uploaded content. + contentLengthRange struct { + min int64 + max int64 + } + + // Post form data. + formData map[string]string +} + +// NewPostPolicy - Instantiate new post policy. +func NewPostPolicy() *PostPolicy { + p := &PostPolicy{} + p.conditions = make([]policyCondition, 0) + p.formData = make(map[string]string) + return p +} + +// SetExpires - Sets expiration time for the new policy. +func (p *PostPolicy) SetExpires(t time.Time) error { + if t.IsZero() { + return ErrInvalidArgument("No expiry time set.") + } + p.expiration = t + return nil +} + +// SetKey - Sets an object name for the policy based upload. +func (p *PostPolicy) SetKey(key string) error { + if strings.TrimSpace(key) == "" || key == "" { + return ErrInvalidArgument("Object name is empty.") + } + policyCond := policyCondition{ + matchType: "eq", + condition: "$key", + value: key, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData["key"] = key + return nil +} + +// SetKeyStartsWith - Sets an object name that an policy based upload +// can start with. +func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error { + if strings.TrimSpace(keyStartsWith) == "" || keyStartsWith == "" { + return ErrInvalidArgument("Object prefix is empty.") + } + policyCond := policyCondition{ + matchType: "starts-with", + condition: "$key", + value: keyStartsWith, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData["key"] = keyStartsWith + return nil +} + +// SetBucket - Sets bucket at which objects will be uploaded to. +func (p *PostPolicy) SetBucket(bucketName string) error { + if strings.TrimSpace(bucketName) == "" || bucketName == "" { + return ErrInvalidArgument("Bucket name is empty.") + } + policyCond := policyCondition{ + matchType: "eq", + condition: "$bucket", + value: bucketName, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData["bucket"] = bucketName + return nil +} + +// SetContentType - Sets content-type of the object for this policy +// based upload. +func (p *PostPolicy) SetContentType(contentType string) error { + if strings.TrimSpace(contentType) == "" || contentType == "" { + return ErrInvalidArgument("No content type specified.") + } + policyCond := policyCondition{ + matchType: "eq", + condition: "$Content-Type", + value: contentType, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData["Content-Type"] = contentType + return nil +} + +// SetContentLengthRange - Set new min and max content length +// condition for all incoming uploads. +func (p *PostPolicy) SetContentLengthRange(min, max int64) error { + if min > max { + return ErrInvalidArgument("Minimum limit is larger than maximum limit.") + } + if min < 0 { + return ErrInvalidArgument("Minimum limit cannot be negative.") + } + if max < 0 { + return ErrInvalidArgument("Maximum limit cannot be negative.") + } + p.contentLengthRange.min = min + p.contentLengthRange.max = max + return nil +} + +// SetSuccessStatusAction - Sets the status success code of the object for this policy +// based upload. +func (p *PostPolicy) SetSuccessStatusAction(status string) error { + if strings.TrimSpace(status) == "" || status == "" { + return ErrInvalidArgument("Status is empty") + } + policyCond := policyCondition{ + matchType: "eq", + condition: "$success_action_status", + value: status, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData["success_action_status"] = status + return nil +} + +// SetUserMetadata - Set user metadata as a key/value couple. +// Can be retrieved through a HEAD request or an event. +func (p *PostPolicy) SetUserMetadata(key string, value string) error { + if strings.TrimSpace(key) == "" || key == "" { + return ErrInvalidArgument("Key is empty") + } + if strings.TrimSpace(value) == "" || value == "" { + return ErrInvalidArgument("Value is empty") + } + headerName := fmt.Sprintf("x-amz-meta-%s", key) + policyCond := policyCondition{ + matchType: "eq", + condition: fmt.Sprintf("$%s", headerName), + value: value, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData[headerName] = value + return nil +} + +// SetUserData - Set user data as a key/value couple. +// Can be retrieved through a HEAD request or an event. +func (p *PostPolicy) SetUserData(key string, value string) error { + if key == "" { + return ErrInvalidArgument("Key is empty") + } + if value == "" { + return ErrInvalidArgument("Value is empty") + } + headerName := fmt.Sprintf("x-amz-%s", key) + policyCond := policyCondition{ + matchType: "eq", + condition: fmt.Sprintf("$%s", headerName), + value: value, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData[headerName] = value + return nil +} + +// addNewPolicy - internal helper to validate adding new policies. +func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error { + if policyCond.matchType == "" || policyCond.condition == "" || policyCond.value == "" { + return ErrInvalidArgument("Policy fields are empty.") + } + p.conditions = append(p.conditions, policyCond) + return nil +} + +// Stringer interface for printing policy in json formatted string. +func (p PostPolicy) String() string { + return string(p.marshalJSON()) +} + +// marshalJSON - Provides Marshalled JSON in bytes. +func (p PostPolicy) marshalJSON() []byte { + expirationStr := `"expiration":"` + p.expiration.Format(expirationDateFormat) + `"` + var conditionsStr string + conditions := []string{} + for _, po := range p.conditions { + conditions = append(conditions, fmt.Sprintf("[\"%s\",\"%s\",\"%s\"]", po.matchType, po.condition, po.value)) + } + if p.contentLengthRange.min != 0 || p.contentLengthRange.max != 0 { + conditions = append(conditions, fmt.Sprintf("[\"content-length-range\", %d, %d]", + p.contentLengthRange.min, p.contentLengthRange.max)) + } + if len(conditions) > 0 { + conditionsStr = `"conditions":[` + strings.Join(conditions, ",") + "]" + } + retStr := "{" + retStr = retStr + expirationStr + "," + retStr = retStr + conditionsStr + retStr = retStr + "}" + return []byte(retStr) +} + +// base64 - Produces base64 of PostPolicy's Marshalled json. +func (p PostPolicy) base64() string { + return base64.StdEncoding.EncodeToString(p.marshalJSON()) +} diff --git a/retry-continous.go b/retry-continous.go new file mode 100644 index 0000000..f31dfa6 --- /dev/null +++ b/retry-continous.go @@ -0,0 +1,69 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import "time" + +// newRetryTimerContinous creates a timer with exponentially increasing delays forever. +func (c Client) newRetryTimerContinous(unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int { + attemptCh := make(chan int) + + // normalize jitter to the range [0, 1.0] + if jitter < NoJitter { + jitter = NoJitter + } + if jitter > MaxJitter { + jitter = MaxJitter + } + + // computes the exponential backoff duration according to + // https://www.awsarchitectureblog.com/2015/03/backoff.html + exponentialBackoffWait := func(attempt int) time.Duration { + // 1< maxAttempt { + attempt = maxAttempt + } + //sleep = random_between(0, min(cap, base * 2 ** attempt)) + sleep := unit * time.Duration(1< cap { + sleep = cap + } + if jitter != NoJitter { + sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter) + } + return sleep + } + + go func() { + defer close(attemptCh) + var nextBackoff int + for { + select { + // Attempts starts. + case attemptCh <- nextBackoff: + nextBackoff++ + case <-doneCh: + // Stop the routine. + return + } + time.Sleep(exponentialBackoffWait(nextBackoff)) + } + }() + return attemptCh +} diff --git a/retry.go b/retry.go new file mode 100644 index 0000000..2a76707 --- /dev/null +++ b/retry.go @@ -0,0 +1,153 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "net" + "net/http" + "net/url" + "strings" + "time" +) + +// MaxRetry is the maximum number of retries before stopping. +var MaxRetry = 10 + +// MaxJitter will randomize over the full exponential backoff time +const MaxJitter = 1.0 + +// NoJitter disables the use of jitter for randomizing the exponential backoff time +const NoJitter = 0.0 + +// DefaultRetryUnit - default unit multiplicative per retry. +// defaults to 1 second. +const DefaultRetryUnit = time.Second + +// DefaultRetryCap - Each retry attempt never waits no longer than +// this maximum time duration. +const DefaultRetryCap = time.Second * 30 + +// newRetryTimer creates a timer with exponentially increasing +// delays until the maximum retry attempts are reached. +func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int { + attemptCh := make(chan int) + + // computes the exponential backoff duration according to + // https://www.awsarchitectureblog.com/2015/03/backoff.html + exponentialBackoffWait := func(attempt int) time.Duration { + // normalize jitter to the range [0, 1.0] + if jitter < NoJitter { + jitter = NoJitter + } + if jitter > MaxJitter { + jitter = MaxJitter + } + + //sleep = random_between(0, min(cap, base * 2 ** attempt)) + sleep := unit * time.Duration(1< cap { + sleep = cap + } + if jitter != NoJitter { + sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter) + } + return sleep + } + + go func() { + defer close(attemptCh) + for i := 0; i < maxRetry; i++ { + select { + // Attempts start from 1. + case attemptCh <- i + 1: + case <-doneCh: + // Stop the routine. + return + } + time.Sleep(exponentialBackoffWait(i)) + } + }() + return attemptCh +} + +// isHTTPReqErrorRetryable - is http requests error retryable, such +// as i/o timeout, connection broken etc.. +func isHTTPReqErrorRetryable(err error) bool { + if err == nil { + return false + } + switch e := err.(type) { + case *url.Error: + switch e.Err.(type) { + case *net.DNSError, *net.OpError, net.UnknownNetworkError: + return true + } + if strings.Contains(err.Error(), "Connection closed by foreign host") { + return true + } else if strings.Contains(err.Error(), "net/http: TLS handshake timeout") { + // If error is - tlsHandshakeTimeoutError, retry. + return true + } else if strings.Contains(err.Error(), "i/o timeout") { + // If error is - tcp timeoutError, retry. + return true + } else if strings.Contains(err.Error(), "connection timed out") { + // If err is a net.Dial timeout, retry. + return true + } else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") { + // If error is transport connection broken, retry. + return true + } + } + return false +} + +// List of AWS S3 error codes which are retryable. +var retryableS3Codes = map[string]struct{}{ + "RequestError": {}, + "RequestTimeout": {}, + "Throttling": {}, + "ThrottlingException": {}, + "RequestLimitExceeded": {}, + "RequestThrottled": {}, + "InternalError": {}, + "ExpiredToken": {}, + "ExpiredTokenException": {}, + "SlowDown": {}, + // Add more AWS S3 codes here. +} + +// isS3CodeRetryable - is s3 error code retryable. +func isS3CodeRetryable(s3Code string) (ok bool) { + _, ok = retryableS3Codes[s3Code] + return ok +} + +// List of HTTP status codes which are retryable. +var retryableHTTPStatusCodes = map[int]struct{}{ + 429: {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet + http.StatusInternalServerError: {}, + http.StatusBadGateway: {}, + http.StatusServiceUnavailable: {}, + // Add more HTTP status codes here. +} + +// isHTTPStatusRetryable - is HTTP error code retryable. +func isHTTPStatusRetryable(httpStatusCode int) (ok bool) { + _, ok = retryableHTTPStatusCodes[httpStatusCode] + return ok +} diff --git a/s3-endpoints.go b/s3-endpoints.go new file mode 100644 index 0000000..0589295 --- /dev/null +++ b/s3-endpoints.go @@ -0,0 +1,50 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +// awsS3EndpointMap Amazon S3 endpoint map. +var awsS3EndpointMap = map[string]string{ + "us-east-1": "s3.amazonaws.com", + "us-east-2": "s3-us-east-2.amazonaws.com", + "us-west-2": "s3-us-west-2.amazonaws.com", + "us-west-1": "s3-us-west-1.amazonaws.com", + "ca-central-1": "s3-ca-central-1.amazonaws.com", + "eu-west-1": "s3-eu-west-1.amazonaws.com", + "eu-west-2": "s3-eu-west-2.amazonaws.com", + "eu-west-3": "s3-eu-west-3.amazonaws.com", + "eu-central-1": "s3-eu-central-1.amazonaws.com", + "ap-south-1": "s3-ap-south-1.amazonaws.com", + "ap-southeast-1": "s3-ap-southeast-1.amazonaws.com", + "ap-southeast-2": "s3-ap-southeast-2.amazonaws.com", + "ap-northeast-1": "s3-ap-northeast-1.amazonaws.com", + "ap-northeast-2": "s3-ap-northeast-2.amazonaws.com", + "sa-east-1": "s3-sa-east-1.amazonaws.com", + "us-gov-west-1": "s3-us-gov-west-1.amazonaws.com", + "cn-north-1": "s3.cn-north-1.amazonaws.com.cn", + "cn-northwest-1": "s3.cn-northwest-1.amazonaws.com.cn", +} + +// getS3Endpoint get Amazon S3 endpoint based on the bucket location. +func getS3Endpoint(bucketLocation string) (s3Endpoint string) { + s3Endpoint, ok := awsS3EndpointMap[bucketLocation] + if !ok { + // Default to 's3.amazonaws.com' endpoint. + s3Endpoint = "s3.amazonaws.com" + } + return s3Endpoint +} diff --git a/s3-error.go b/s3-error.go new file mode 100644 index 0000000..3b11776 --- /dev/null +++ b/s3-error.go @@ -0,0 +1,61 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +// Non exhaustive list of AWS S3 standard error responses - +// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html +var s3ErrorResponseMap = map[string]string{ + "AccessDenied": "Access Denied.", + "BadDigest": "The Content-Md5 you specified did not match what we received.", + "EntityTooSmall": "Your proposed upload is smaller than the minimum allowed object size.", + "EntityTooLarge": "Your proposed upload exceeds the maximum allowed object size.", + "IncompleteBody": "You did not provide the number of bytes specified by the Content-Length HTTP header.", + "InternalError": "We encountered an internal error, please try again.", + "InvalidAccessKeyId": "The access key ID you provided does not exist in our records.", + "InvalidBucketName": "The specified bucket is not valid.", + "InvalidDigest": "The Content-Md5 you specified is not valid.", + "InvalidRange": "The requested range is not satisfiable", + "MalformedXML": "The XML you provided was not well-formed or did not validate against our published schema.", + "MissingContentLength": "You must provide the Content-Length HTTP header.", + "MissingContentMD5": "Missing required header for this request: Content-Md5.", + "MissingRequestBodyError": "Request body is empty.", + "NoSuchBucket": "The specified bucket does not exist.", + "NoSuchBucketPolicy": "The bucket policy does not exist", + "NoSuchKey": "The specified key does not exist.", + "NoSuchUpload": "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.", + "NotImplemented": "A header you provided implies functionality that is not implemented", + "PreconditionFailed": "At least one of the pre-conditions you specified did not hold", + "RequestTimeTooSkewed": "The difference between the request time and the server's time is too large.", + "SignatureDoesNotMatch": "The request signature we calculated does not match the signature you provided. Check your key and signing method.", + "MethodNotAllowed": "The specified method is not allowed against this resource.", + "InvalidPart": "One or more of the specified parts could not be found.", + "InvalidPartOrder": "The list of parts was not in ascending order. The parts list must be specified in order by part number.", + "InvalidObjectState": "The operation is not valid for the current state of the object.", + "AuthorizationHeaderMalformed": "The authorization header is malformed; the region is wrong.", + "MalformedPOSTRequest": "The body of your POST request is not well-formed multipart/form-data.", + "BucketNotEmpty": "The bucket you tried to delete is not empty", + "AllAccessDisabled": "All access to this bucket has been disabled.", + "MalformedPolicy": "Policy has invalid resource.", + "MissingFields": "Missing fields in request.", + "AuthorizationQueryParametersError": "Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting \"/YYYYMMDD/REGION/SERVICE/aws4_request\".", + "MalformedDate": "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.", + "BucketAlreadyOwnedByYou": "Your previous request to create the named bucket succeeded and you already own it.", + "InvalidDuration": "Duration provided in the request is invalid.", + "XAmzContentSHA256Mismatch": "The provided 'x-amz-content-sha256' header does not match what was computed.", + // Add new API errors here. +} diff --git a/test-utils_test.go b/test-utils_test.go new file mode 100644 index 0000000..6f6443c --- /dev/null +++ b/test-utils_test.go @@ -0,0 +1,75 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "encoding/xml" + "io/ioutil" + "net/http" + "strconv" +) + +// Contains common used utilities for tests. + +// APIError Used for mocking error response from server. +type APIError struct { + Code string + Description string + HTTPStatusCode int +} + +// Mocks XML error response from the server. +func generateErrorResponse(resp *http.Response, APIErr APIError, bucketName string) *http.Response { + // generate error response. + errorResponse := getAPIErrorResponse(APIErr, bucketName) + encodedErrorResponse := encodeResponse(errorResponse) + // write Header. + resp.StatusCode = APIErr.HTTPStatusCode + resp.Body = ioutil.NopCloser(bytes.NewBuffer(encodedErrorResponse)) + + return resp +} + +// getErrorResponse gets in standard error and resource value and +// provides a encodable populated response values. +func getAPIErrorResponse(err APIError, bucketName string) ErrorResponse { + var errResp = ErrorResponse{} + errResp.Code = err.Code + errResp.Message = err.Description + errResp.BucketName = bucketName + return errResp +} + +// Encodes the response headers into XML format. +func encodeResponse(response interface{}) []byte { + var bytesBuffer bytes.Buffer + bytesBuffer.WriteString(xml.Header) + encode := xml.NewEncoder(&bytesBuffer) + encode.Encode(response) + return bytesBuffer.Bytes() +} + +// Convert string to bool and always return false if any error +func mustParseBool(str string) bool { + b, err := strconv.ParseBool(str) + if err != nil { + return false + } + return b +} diff --git a/transport.go b/transport.go new file mode 100644 index 0000000..88700cf --- /dev/null +++ b/transport.go @@ -0,0 +1,50 @@ +// +build go1.7 go1.8 + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017-2018 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "net" + "net/http" + "time" +) + +// DefaultTransport - this default transport is similar to +// http.DefaultTransport but with additional param DisableCompression +// is set to true to avoid decompressing content with 'gzip' encoding. +var DefaultTransport http.RoundTripper = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + // Set this value so that the underlying transport round-tripper + // doesn't try to auto decode the body of objects with + // content-encoding set to `gzip`. + // + // Refer: + // https://golang.org/src/net/http/transport.go?h=roundTrip#L1843 + DisableCompression: true, +} diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..8483f38 --- /dev/null +++ b/utils.go @@ -0,0 +1,272 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "crypto/md5" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// xmlDecoder provide decoded value in xml. +func xmlDecoder(body io.Reader, v interface{}) error { + d := xml.NewDecoder(body) + return d.Decode(v) +} + +// sum256 calculate sha256sum for an input byte array, returns hex encoded. +func sum256Hex(data []byte) string { + hash := sha256.New() + hash.Write(data) + return hex.EncodeToString(hash.Sum(nil)) +} + +// sumMD5Base64 calculate md5sum for an input byte array, returns base64 encoded. +func sumMD5Base64(data []byte) string { + hash := md5.New() + hash.Write(data) + return base64.StdEncoding.EncodeToString(hash.Sum(nil)) +} + +// getEndpointURL - construct a new endpoint. +func getEndpointURL(endpoint string, secure bool) (*url.URL, error) { + if strings.Contains(endpoint, ":") { + host, _, err := net.SplitHostPort(endpoint) + if err != nil { + return nil, err + } + if !s3utils.IsValidIP(host) && !s3utils.IsValidDomain(host) { + msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards." + return nil, ErrInvalidArgument(msg) + } + } else { + if !s3utils.IsValidIP(endpoint) && !s3utils.IsValidDomain(endpoint) { + msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards." + return nil, ErrInvalidArgument(msg) + } + } + // If secure is false, use 'http' scheme. + scheme := "https" + if !secure { + scheme = "http" + } + + // Construct a secured endpoint URL. + endpointURLStr := scheme + "://" + endpoint + endpointURL, err := url.Parse(endpointURLStr) + if err != nil { + return nil, err + } + + // Validate incoming endpoint URL. + if err := isValidEndpointURL(*endpointURL); err != nil { + return nil, err + } + return endpointURL, nil +} + +// closeResponse close non nil response with any response Body. +// convenient wrapper to drain any remaining data on response body. +// +// Subsequently this allows golang http RoundTripper +// to re-use the same connection for future requests. +func closeResponse(resp *http.Response) { + // Callers should close resp.Body when done reading from it. + // If resp.Body is not closed, the Client's underlying RoundTripper + // (typically Transport) may not be able to re-use a persistent TCP + // connection to the server for a subsequent "keep-alive" request. + if resp != nil && resp.Body != nil { + // Drain any remaining Body and then close the connection. + // Without this closing connection would disallow re-using + // the same connection for future uses. + // - http://stackoverflow.com/a/17961593/4465767 + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + } +} + +var ( + // Hex encoded string of nil sha256sum bytes. + emptySHA256Hex = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + // Sentinel URL is the default url value which is invalid. + sentinelURL = url.URL{} +) + +// Verify if input endpoint URL is valid. +func isValidEndpointURL(endpointURL url.URL) error { + if endpointURL == sentinelURL { + return ErrInvalidArgument("Endpoint url cannot be empty.") + } + if endpointURL.Path != "/" && endpointURL.Path != "" { + return ErrInvalidArgument("Endpoint url cannot have fully qualified paths.") + } + if strings.Contains(endpointURL.Host, ".s3.amazonaws.com") { + if !s3utils.IsAmazonEndpoint(endpointURL) { + return ErrInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'.") + } + } + if strings.Contains(endpointURL.Host, ".googleapis.com") { + if !s3utils.IsGoogleEndpoint(endpointURL) { + return ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.") + } + } + return nil +} + +// Verify if input expires value is valid. +func isValidExpiry(expires time.Duration) error { + expireSeconds := int64(expires / time.Second) + if expireSeconds < 1 { + return ErrInvalidArgument("Expires cannot be lesser than 1 second.") + } + if expireSeconds > 604800 { + return ErrInvalidArgument("Expires cannot be greater than 7 days.") + } + return nil +} + +// make a copy of http.Header +func cloneHeader(h http.Header) http.Header { + h2 := make(http.Header, len(h)) + for k, vv := range h { + vv2 := make([]string, len(vv)) + copy(vv2, vv) + h2[k] = vv2 + } + return h2 +} + +// Filter relevant response headers from +// the HEAD, GET http response. The function takes +// a list of headers which are filtered out and +// returned as a new http header. +func filterHeader(header http.Header, filterKeys []string) (filteredHeader http.Header) { + filteredHeader = cloneHeader(header) + for _, key := range filterKeys { + filteredHeader.Del(key) + } + return filteredHeader +} + +// regCred matches credential string in HTTP header +var regCred = regexp.MustCompile("Credential=([A-Z0-9]+)/") + +// regCred matches signature string in HTTP header +var regSign = regexp.MustCompile("Signature=([[0-9a-f]+)") + +// Redact out signature value from authorization string. +func redactSignature(origAuth string) string { + if !strings.HasPrefix(origAuth, signV4Algorithm) { + // Set a temporary redacted auth + return "AWS **REDACTED**:**REDACTED**" + } + + /// Signature V4 authorization header. + + // Strip out accessKeyID from: + // Credential=////aws4_request + newAuth := regCred.ReplaceAllString(origAuth, "Credential=**REDACTED**/") + + // Strip out 256-bit signature from: Signature=<256-bit signature> + return regSign.ReplaceAllString(newAuth, "Signature=**REDACTED**") +} + +// Get default location returns the location based on the input +// URL `u`, if region override is provided then all location +// defaults to regionOverride. +// +// If no other cases match then the location is set to `us-east-1` +// as a last resort. +func getDefaultLocation(u url.URL, regionOverride string) (location string) { + if regionOverride != "" { + return regionOverride + } + region := s3utils.GetRegionFromURL(u) + if region == "" { + region = "us-east-1" + } + return region +} + +var supportedHeaders = []string{ + "content-type", + "cache-control", + "content-encoding", + "content-disposition", + "content-language", + "x-amz-website-redirect-location", + "expires", + // Add more supported headers here. +} + +// isStorageClassHeader returns true if the header is a supported storage class header +func isStorageClassHeader(headerKey string) bool { + return strings.ToLower(amzStorageClass) == strings.ToLower(headerKey) +} + +// isStandardHeader returns true if header is a supported header and not a custom header +func isStandardHeader(headerKey string) bool { + key := strings.ToLower(headerKey) + for _, header := range supportedHeaders { + if strings.ToLower(header) == key { + return true + } + } + return false +} + +// sseHeaders is list of server side encryption headers +var sseHeaders = []string{ + "x-amz-server-side-encryption", + "x-amz-server-side-encryption-aws-kms-key-id", + "x-amz-server-side-encryption-context", + "x-amz-server-side-encryption-customer-algorithm", + "x-amz-server-side-encryption-customer-key", + "x-amz-server-side-encryption-customer-key-MD5", +} + +// isSSEHeader returns true if header is a server side encryption header. +func isSSEHeader(headerKey string) bool { + key := strings.ToLower(headerKey) + for _, h := range sseHeaders { + if strings.ToLower(h) == key { + return true + } + } + return false +} + +// isAmzHeader returns true if header is a x-amz-meta-* or x-amz-acl header. +func isAmzHeader(headerKey string) bool { + key := strings.ToLower(headerKey) + + return strings.HasPrefix(key, "x-amz-meta-") || strings.HasPrefix(key, "x-amz-grant-") || key == "x-amz-acl" || isSSEHeader(headerKey) +} diff --git a/utils_test.go b/utils_test.go new file mode 100644 index 0000000..2e60f77 --- /dev/null +++ b/utils_test.go @@ -0,0 +1,378 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "fmt" + "net/http" + "net/url" + "testing" + "time" + + "github.com/minio/minio-go/pkg/s3utils" +) + +// Tests signature redacting function used +// in filtering on-wire Authorization header. +func TestRedactSignature(t *testing.T) { + testCases := []struct { + authValue string + expectedRedactedAuthValue string + }{ + { + authValue: "AWS 1231313:888x000231==", + expectedRedactedAuthValue: "AWS **REDACTED**:**REDACTED**", + }, + { + authValue: "AWS4-HMAC-SHA256 Credential=12312313/20170613/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=02131231312313213", + expectedRedactedAuthValue: "AWS4-HMAC-SHA256 Credential=**REDACTED**/20170613/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=**REDACTED**", + }, + } + + for i, testCase := range testCases { + redactedAuthValue := redactSignature(testCase.authValue) + if redactedAuthValue != testCase.expectedRedactedAuthValue { + t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedRedactedAuthValue, redactedAuthValue) + } + } +} + +// Tests filter header function by filtering out +// some custom header keys. +func TestFilterHeader(t *testing.T) { + header := http.Header{} + header.Set("Content-Type", "binary/octet-stream") + header.Set("Content-Encoding", "gzip") + newHeader := filterHeader(header, []string{"Content-Type"}) + if len(newHeader) > 1 { + t.Fatalf("Unexpected size of the returned header, should be 1, got %d", len(newHeader)) + } + if newHeader.Get("Content-Encoding") != "gzip" { + t.Fatalf("Unexpected content-encoding value, expected 'gzip', got %s", newHeader.Get("Content-Encoding")) + } +} + +// Tests for 'getEndpointURL(endpoint string, inSecure bool)'. +func TestGetEndpointURL(t *testing.T) { + testCases := []struct { + // Inputs. + endPoint string + secure bool + + // Expected result. + result string + err error + // Flag indicating whether the test is expected to pass or not. + shouldPass bool + }{ + {"s3.amazonaws.com", true, "https://s3.amazonaws.com", nil, true}, + {"s3.cn-north-1.amazonaws.com.cn", true, "https://s3.cn-north-1.amazonaws.com.cn", nil, true}, + {"s3.cn-northwest-1.amazonaws.com.cn", true, "https://s3.cn-northwest-1.amazonaws.com.cn", nil, true}, + {"s3.amazonaws.com", false, "http://s3.amazonaws.com", nil, true}, + {"s3.cn-north-1.amazonaws.com.cn", false, "http://s3.cn-north-1.amazonaws.com.cn", nil, true}, + {"s3.cn-northwest-1.amazonaws.com.cn", false, "http://s3.cn-northwest-1.amazonaws.com.cn", nil, true}, + {"192.168.1.1:9000", false, "http://192.168.1.1:9000", nil, true}, + {"192.168.1.1:9000", true, "https://192.168.1.1:9000", nil, true}, + {"s3.amazonaws.com:443", true, "https://s3.amazonaws.com:443", nil, true}, + {"13333.123123.-", true, "", ErrInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-")), false}, + {"13333.123123.-", true, "", ErrInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-")), false}, + {"storage.googleapis.com:4000", true, "", ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'."), false}, + {"s3.aamzza.-", true, "", ErrInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "s3.aamzza.-")), false}, + {"", true, "", ErrInvalidArgument("Endpoint: does not follow ip address or domain name standards."), false}, + } + + for i, testCase := range testCases { + result, err := getEndpointURL(testCase.endPoint, testCase.secure) + if err != nil && testCase.shouldPass { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) + } + if err == nil && !testCase.shouldPass { + t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if err.Error() != testCase.err.Error() { + t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) + } + } + + // Test passes as expected, but the output values are verified for correctness here. + if err == nil && testCase.shouldPass { + if testCase.result != result.String() { + t.Errorf("Test %d: Expected the result Url to be \"%s\", but found \"%s\" instead", i+1, testCase.result, result.String()) + } + } + } +} + +// Tests validate end point validator. +func TestIsValidEndpointURL(t *testing.T) { + testCases := []struct { + url string + err error + // Flag indicating whether the test is expected to pass or not. + shouldPass bool + }{ + {"", ErrInvalidArgument("Endpoint url cannot be empty."), false}, + {"/", nil, true}, + {"https://s3.amazonaws.com", nil, true}, + {"https://s3.cn-north-1.amazonaws.com.cn", nil, true}, + {"https://s3-us-gov-west-1.amazonaws.com", nil, true}, + {"https://s3-fips-us-gov-west-1.amazonaws.com", nil, true}, + {"https://s3.amazonaws.com/", nil, true}, + {"https://storage.googleapis.com/", nil, true}, + {"https://z3.amazonaws.com", nil, true}, + {"https://mybalancer.us-east-1.elb.amazonaws.com", nil, true}, + {"192.168.1.1", ErrInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, + {"https://amazon.googleapis.com/", ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'."), false}, + {"https://storage.googleapis.com/bucket/", ErrInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, + {"https://s3.amazonaws.com/bucket/object", ErrInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, + } + + for i, testCase := range testCases { + var u url.URL + if testCase.url == "" { + u = sentinelURL + } else { + u1, err := url.Parse(testCase.url) + if err != nil { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err) + } + u = *u1 + } + err := isValidEndpointURL(u) + if err != nil && testCase.shouldPass { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err) + } + if err == nil && !testCase.shouldPass { + t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if err.Error() != testCase.err.Error() { + t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err, err) + } + } + + } +} + +func TestDefaultBucketLocation(t *testing.T) { + testCases := []struct { + endpointURL url.URL + regionOverride string + expectedLocation string + }{ + // Region override is set URL is ignored. - Test 1. + { + endpointURL: url.URL{Host: "s3-fips-us-gov-west-1.amazonaws.com"}, + regionOverride: "us-west-1", + expectedLocation: "us-west-1", + }, + // No region override, url based preferenced is honored - Test 2. + { + endpointURL: url.URL{Host: "s3-fips-us-gov-west-1.amazonaws.com"}, + regionOverride: "", + expectedLocation: "us-gov-west-1", + }, + // Region override is honored - Test 3. + { + endpointURL: url.URL{Host: "s3.amazonaws.com"}, + regionOverride: "us-west-1", + expectedLocation: "us-west-1", + }, + // China region should be honored, region override not provided. - Test 4. + { + endpointURL: url.URL{Host: "s3.cn-north-1.amazonaws.com.cn"}, + regionOverride: "", + expectedLocation: "cn-north-1", + }, + // China region should be honored, region override not provided. - Test 5. + { + endpointURL: url.URL{Host: "s3.cn-northwest-1.amazonaws.com.cn"}, + regionOverride: "", + expectedLocation: "cn-northwest-1", + }, + // No region provided, no standard region strings provided as well. - Test 6. + { + endpointURL: url.URL{Host: "s3.amazonaws.com"}, + regionOverride: "", + expectedLocation: "us-east-1", + }, + } + + for i, testCase := range testCases { + retLocation := getDefaultLocation(testCase.endpointURL, testCase.regionOverride) + if testCase.expectedLocation != retLocation { + t.Errorf("Test %d: Expected location %s, got %s", i+1, testCase.expectedLocation, retLocation) + } + } +} + +// Tests validate the expiry time validator. +func TestIsValidExpiry(t *testing.T) { + testCases := []struct { + // Input. + duration time.Duration + // Expected result. + err error + // Flag to indicate whether the test should pass. + shouldPass bool + }{ + {100 * time.Millisecond, ErrInvalidArgument("Expires cannot be lesser than 1 second."), false}, + {604801 * time.Second, ErrInvalidArgument("Expires cannot be greater than 7 days."), false}, + {0 * time.Second, ErrInvalidArgument("Expires cannot be lesser than 1 second."), false}, + {1 * time.Second, nil, true}, + {10000 * time.Second, nil, true}, + {999 * time.Second, nil, true}, + } + + for i, testCase := range testCases { + err := isValidExpiry(testCase.duration) + if err != nil && testCase.shouldPass { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) + } + if err == nil && !testCase.shouldPass { + t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if err.Error() != testCase.err.Error() { + t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) + } + } + + } +} + +// Tests validate the bucket name validator. +func TestIsValidBucketName(t *testing.T) { + testCases := []struct { + // Input. + bucketName string + // Expected result. + err error + // Flag to indicate whether test should Pass. + shouldPass bool + }{ + {".mybucket", ErrInvalidBucketName("Bucket name contains invalid characters"), false}, + {"mybucket.", ErrInvalidBucketName("Bucket name contains invalid characters"), false}, + {"mybucket-", ErrInvalidBucketName("Bucket name contains invalid characters"), false}, + {"my", ErrInvalidBucketName("Bucket name cannot be smaller than 3 characters"), false}, + {"", ErrInvalidBucketName("Bucket name cannot be empty"), false}, + {"my..bucket", ErrInvalidBucketName("Bucket name contains invalid characters"), false}, + {"my.bucket.com", nil, true}, + {"my-bucket", nil, true}, + {"123my-bucket", nil, true}, + } + + for i, testCase := range testCases { + err := s3utils.CheckValidBucketName(testCase.bucketName) + if err != nil && testCase.shouldPass { + t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) + } + if err == nil && !testCase.shouldPass { + t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if err.Error() != testCase.err.Error() { + t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) + } + } + + } + +} + +// Tests if header is standard supported header +func TestIsStandardHeader(t *testing.T) { + testCases := []struct { + // Input. + header string + // Expected result. + expectedValue bool + }{ + {"content-encoding", true}, + {"content-type", true}, + {"cache-control", true}, + {"content-disposition", true}, + {"content-language", true}, + {"random-header", false}, + } + + for i, testCase := range testCases { + actual := isStandardHeader(testCase.header) + if actual != testCase.expectedValue { + t.Errorf("Test %d: Expected to pass, but failed", i+1) + } + } + +} + +// Tests if header is server encryption header +func TestIsSSEHeader(t *testing.T) { + testCases := []struct { + // Input. + header string + // Expected result. + expectedValue bool + }{ + {"x-amz-server-side-encryption", true}, + {"x-amz-server-side-encryption-aws-kms-key-id", true}, + {"x-amz-server-side-encryption-context", true}, + {"x-amz-server-side-encryption-customer-algorithm", true}, + {"x-amz-server-side-encryption-customer-key", true}, + {"x-amz-server-side-encryption-customer-key-MD5", true}, + {"random-header", false}, + } + + for i, testCase := range testCases { + actual := isSSEHeader(testCase.header) + if actual != testCase.expectedValue { + t.Errorf("Test %d: Expected to pass, but failed", i+1) + } + } +} + +// Tests if header is x-amz-meta or x-amz-acl +func TestIsAmzHeader(t *testing.T) { + testCases := []struct { + // Input. + header string + // Expected result. + expectedValue bool + }{ + {"x-amz-iv", false}, + {"x-amz-key", false}, + {"x-amz-matdesc", false}, + {"x-amz-meta-x-amz-iv", true}, + {"x-amz-meta-x-amz-key", true}, + {"x-amz-meta-x-amz-matdesc", true}, + {"x-amz-acl", true}, + {"random-header", false}, + } + + for i, testCase := range testCases { + actual := isAmzHeader(testCase.header) + if actual != testCase.expectedValue { + t.Errorf("Test %d: Expected to pass, but failed", i+1) + } + } + +} -- cgit v1.2.3 From 3cfd38af7dc586b8dc2fbc590805f2fa3b3f8672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Sipma?= Date: Sat, 8 Dec 2018 10:31:59 +0100 Subject: Import golang-github-minio-minio-go_6.0.11-2.debian.tar.xz [dgit import tarball golang-github-minio-minio-go 6.0.11-2 golang-github-minio-minio-go_6.0.11-2.debian.tar.xz] --- changelog | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ compat | 1 + control | 29 ++++++++++++++++++++++ copyright | 27 +++++++++++++++++++++ gbp.conf | 5 ++++ gitlab-ci.yml | 28 +++++++++++++++++++++ rules | 13 ++++++++++ source/format | 1 + watch | 4 +++ 9 files changed, 186 insertions(+) create mode 100644 changelog create mode 100644 compat create mode 100644 control create mode 100644 copyright create mode 100644 gbp.conf create mode 100644 gitlab-ci.yml create mode 100755 rules create mode 100644 source/format create mode 100644 watch diff --git a/changelog b/changelog new file mode 100644 index 0000000..cb5fb53 --- /dev/null +++ b/changelog @@ -0,0 +1,78 @@ +golang-github-minio-minio-go (6.0.11-2) unstable; urgency=medium + + * fix messing up in upstream & pristine-tar branches + + -- Félix Sipma Sat, 08 Dec 2018 10:31:59 +0100 + +golang-github-minio-minio-go (6.0.11-1) unstable; urgency=medium + + * New upstream version 6.0.11 (Closes: #915838) + * bump Standards-Version to 4.2.1 (no change required) + + -- Félix Sipma Sat, 08 Dec 2018 10:16:34 +0100 + +golang-github-minio-minio-go (6.0.1-1) unstable; urgency=medium + + * New upstream release. + * (Build-)Depends: + + golang-golang-x-crypto-dev + + golang-golang-x-net-dev + * Standards-Version: 4.1.4. + + -- Dmitry Smirnov Fri, 01 Jun 2018 09:46:06 +1000 + +golang-github-minio-minio-go (4.0.5-3) unstable; urgency=medium + + * Team upload. + * Vcs-* urls: pkg-go-team -> go-team. + + -- Alexandre Viau Tue, 06 Feb 2018 00:15:38 -0500 + +golang-github-minio-minio-go (4.0.5-2) unstable; urgency=medium + + * Team upload. + * point Vcs-* urls to salsa.d.o subproject + + -- Alexandre Viau Thu, 25 Jan 2018 16:53:25 -0500 + +golang-github-minio-minio-go (4.0.5-1) unstable; urgency=medium + + * New upstream version 4.0.5 + * remove go-homedir patch (upstream now uses github.com/mitchellh/go-homedir) + * move to salsa.debian.org + * update Standards-Version to 4.1.3 + - use priority optional instead of extra + - use https URL in copyright file + + -- Félix Sipma Fri, 05 Jan 2018 11:29:36 +0100 + +golang-github-minio-minio-go (3.0.0-1) unstable; urgency=medium + + * gbp.conf: remove merge = false for idiomatic repository layout + * New upstream version 3.0.0 + * Experimental to unstable, as the freeze is over + + -- Michael Stapelberg Fri, 21 Jul 2017 08:05:09 +0200 + +golang-github-minio-minio-go (2.1.0-1) experimental; urgency=medium + + * New upstream version 2.1.0 + * avoid tests which access to s3.amazonaws.com with '-short' flag + + -- Félix Sipma Wed, 17 May 2017 11:20:05 +0200 + +golang-github-minio-minio-go (2.0.2-1) unstable; urgency=medium + + * New upstream version 2.0.2 + * restore dh-make-golang default gbp.conf + * increase debian/compat to 10 + * update debian/control + * improve package description + + -- Félix Sipma Sun, 11 Dec 2016 23:48:22 +0100 + +golang-github-minio-minio-go (2.0.1-1) unstable; urgency=medium + + * Initial release (Closes: #831837). + + -- Dmitry Smirnov Wed, 20 Jul 2016 13:42:58 +1000 diff --git a/compat b/compat new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/compat @@ -0,0 +1 @@ +10 diff --git a/control b/control new file mode 100644 index 0000000..d431b2f --- /dev/null +++ b/control @@ -0,0 +1,29 @@ +Source: golang-github-minio-minio-go +Section: devel +Priority: optional +Maintainer: Debian Go Packaging Team +Uploaders: Dmitry Smirnov , Félix Sipma +Build-Depends: debhelper (>= 10), + dh-golang, + golang-any + ,golang-github-go-ini-ini-dev + ,golang-github-mitchellh-go-homedir-dev + ,golang-golang-x-crypto-dev + ,golang-golang-x-net-dev +Standards-Version: 4.2.1 +Homepage: https://github.com/minio/minio-go +Vcs-Browser: https://salsa.debian.org/go-team/packages/golang-github-minio-minio-go +Vcs-Git: https://salsa.debian.org/go-team/packages/golang-github-minio-minio-go.git +XS-Go-Import-Path: github.com/minio/minio-go + +Package: golang-github-minio-minio-go-dev +Architecture: all +Depends: ${shlibs:Depends}, + ${misc:Depends} + ,golang-github-go-ini-ini-dev + ,golang-github-mitchellh-go-homedir-dev + ,golang-golang-x-crypto-dev + ,golang-golang-x-net-dev +Description: Go library for Amazon S3 compatible cloud storage + The Minio Golang Client SDK provides simple APIs to access any Amazon S3 + compatible object storage server. diff --git a/copyright b/copyright new file mode 100644 index 0000000..85bb367 --- /dev/null +++ b/copyright @@ -0,0 +1,27 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: minio-go +Source: https://github.com/minio/minio-go + +Files: * +Copyright: 2015-2018 Minio, Inc +License: Apache-2.0 + +Files: debian/* +Copyright: 2016-2018 Dmitry Smirnov +License: Apache-2.0 + +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian systems, the complete text of the Apache version 2.0 license + can be found in "/usr/share/common-licenses/Apache-2.0". diff --git a/gbp.conf b/gbp.conf new file mode 100644 index 0000000..1d6ee7b --- /dev/null +++ b/gbp.conf @@ -0,0 +1,5 @@ +[DEFAULT] +pristine-tar = True + +[dch] +id-length = 0 diff --git a/gitlab-ci.yml b/gitlab-ci.yml new file mode 100644 index 0000000..5c8c31b --- /dev/null +++ b/gitlab-ci.yml @@ -0,0 +1,28 @@ + +# auto-generated, DO NOT MODIFY. +# The authoritative copy of this file lives at: +# https://salsa.debian.org/go-team/ci/blob/master/cmd/ci/gitlabciyml.go + +# TODO: publish under debian-go-team/ci +image: stapelberg/ci2 + +test_the_archive: + artifacts: + paths: + - before-applying-commit.json + - after-applying-commit.json + script: + # Create an overlay to discard writes to /srv/gopath/src after the build: + - "rm -rf /cache/overlay/{upper,work}" + - "mkdir -p /cache/overlay/{upper,work}" + - "mount -t overlay overlay -o lowerdir=/srv/gopath/src,upperdir=/cache/overlay/upper,workdir=/cache/overlay/work /srv/gopath/src" + - "export GOPATH=/srv/gopath" + - "export GOCACHE=/cache/go" + # Build the world as-is: + - "ci-build -exemptions=/var/lib/ci-build/exemptions.json > before-applying-commit.json" + # Copy this package into the overlay: + - "GBP_CONF_FILES=:debian/gbp.conf gbp buildpackage --git-no-pristine-tar --git-ignore-branch --git-ignore-new --git-export-dir=/tmp/export --git-no-overlay --git-tarball-dir=/nonexistant --git-cleaner=/bin/true --git-builder='dpkg-buildpackage -S -d --no-sign'" + - "pgt-gopath -dsc /tmp/export/*.dsc" + # Rebuild the world: + - "ci-build -exemptions=/var/lib/ci-build/exemptions.json > after-applying-commit.json" + - "ci-diff before-applying-commit.json after-applying-commit.json" diff --git a/rules b/rules new file mode 100755 index 0000000..bd19989 --- /dev/null +++ b/rules @@ -0,0 +1,13 @@ +#!/usr/bin/make -f + +export DH_GOLANG_EXCLUDES := examples + +# Required for tests: +export DH_GOLANG_INSTALL_EXTRA := ./pkg/credentials/config.json.sample ./pkg/credentials/credentials.sample + +%: + dh $@ --buildsystem=golang --with=golang +override_dh_auto_test: + # set -short param to avoid tests that access to s3.amazonaws.com + # api_functional_v4_test.go is needed for other tests to run + dh_auto_test -- -short diff --git a/source/format b/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/watch b/watch new file mode 100644 index 0000000..011c3bb --- /dev/null +++ b/watch @@ -0,0 +1,4 @@ +version=3 + + https://github.com/minio/minio-go/releases \ + .*/archive/v?(\d[\d\.\-rc]+)\.tar\.gz -- cgit v1.2.3