diff options
Diffstat (limited to 'api-list.go')
| -rw-r--r-- | api-list.go | 223 |
1 files changed, 157 insertions, 66 deletions
diff --git a/api-list.go b/api-list.go index 04f7573..09c1565 100644 --- a/api-list.go +++ b/api-list.go @@ -1,6 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 Minio, Inc. + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import ( "net/url" "strings" - "github.com/minio/minio-go/pkg/s3utils" + "github.com/minio/minio-go/v6/pkg/s3utils" ) // ListBuckets list all buckets owned by this authenticated user. @@ -60,9 +60,13 @@ func (c Client) ListBuckets() ([]BucketInfo, error) { /// Bucket Read Operations. -// ListObjectsV2 lists all objects matching the objectPrefix from -// the specified bucket. If recursion is enabled it would list -// all subdirectories and all its contents. +// ListObjectsV2WithMetadata lists all objects matching the objectPrefix +// from the specified bucket. If recursion is enabled it would list +// all subdirectories and all its contents. This call adds +// UserMetadata information as well for each object. +// +// This is a MinIO extension, this will not work against other S3 +// compatible object storage vendors. // // Your input parameters are just bucketName, objectPrefix, recursive // and a done channel for pro-actively closing the internal go @@ -76,11 +80,18 @@ func (c Client) ListBuckets() ([]BucketInfo, error) { // defer close(doneCh) // // Recursively list all objects in 'mytestbucket' // recursive := true -// for message := range api.ListObjectsV2("mytestbucket", "starthere", recursive, doneCh) { +// // Add metadata +// metadata := true +// for message := range api.ListObjectsV2WithMetadata("mytestbucket", "starthere", recursive, doneCh) { // fmt.Println(message) // } // -func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { +func (c Client) ListObjectsV2WithMetadata(bucketName, objectPrefix string, recursive bool, + doneCh <-chan struct{}) <-chan ObjectInfo { + return c.listObjectsV2(bucketName, objectPrefix, recursive, true, doneCh) +} + +func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metadata bool, doneCh <-chan struct{}) <-chan ObjectInfo { // Allocate new list objects channel. objectStatCh := make(chan ObjectInfo, 1) // Default listing is delimited at "/" @@ -118,7 +129,8 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d var continuationToken string for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, 1000, "") + result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, + fetchOwner, metadata, delimiter, 0, "") if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -142,10 +154,7 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d for _, obj := range result.CommonPrefixes { select { // Send object prefixes. - case objectStatCh <- ObjectInfo{ - Key: obj.Prefix, - Size: 0, - }: + case objectStatCh <- ObjectInfo{Key: obj.Prefix}: // If receives done from the caller, return here. case <-doneCh: return @@ -166,6 +175,30 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d return objectStatCh } +// ListObjectsV2 lists all objects matching the objectPrefix from +// the specified bucket. If recursion is enabled it would list +// all subdirectories and all its contents. +// +// Your input parameters are just bucketName, objectPrefix, recursive +// and a done channel for pro-actively closing the internal go +// routine. If you enable recursive as 'true' this function will +// return back all the objects in a given bucket name and object +// prefix. +// +// api := client.New(....) +// // Create a done channel. +// doneCh := make(chan struct{}) +// defer close(doneCh) +// // Recursively list all objects in 'mytestbucket' +// recursive := true +// for message := range api.ListObjectsV2("mytestbucket", "starthere", recursive, doneCh) { +// fmt.Println(message) +// } +// +func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, doneCh <-chan struct{}) <-chan ObjectInfo { + return c.listObjectsV2(bucketName, objectPrefix, recursive, false, doneCh) +} + // listObjectsV2Query - (List Objects V2) - List some or all (up to 1000) of the objects in a bucket. // // You can use the request parameters as selection criteria to return a subset of the objects in a bucket. @@ -176,7 +209,8 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. // ?start-after - Specifies the key to start after when listing objects in a bucket. -func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { +// ?metadata - Specifies if we want metadata for the objects as part of list operation. +func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketV2Result{}, err @@ -192,30 +226,33 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s // Always set list-type in ListObjects V2 urlValues.Set("list-type", "2") - // Set object prefix. - if objectPrefix != "" { - urlValues.Set("prefix", objectPrefix) + if metadata { + urlValues.Set("metadata", "true") } + + // Always set encoding-type in ListObjects V2 + urlValues.Set("encoding-type", "url") + + // Set object prefix, prefix value to be set to empty is okay. + urlValues.Set("prefix", objectPrefix) + + // Set delimiter, delimiter value to be set to empty is okay. + urlValues.Set("delimiter", delimiter) + // Set continuation token if continuationToken != "" { urlValues.Set("continuation-token", continuationToken) } - // Set delimiter. - if delimiter != "" { - urlValues.Set("delimiter", delimiter) - } // Fetch owner when listing if fetchOwner { urlValues.Set("fetch-owner", "true") } - // maxkeys should default to 1000 or less. - if maxkeys == 0 || maxkeys > 1000 { - maxkeys = 1000 - } // Set max keys. - urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + if maxkeys > 0 { + urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + } // Set start-after if startAfter != "" { @@ -250,6 +287,20 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s return listBucketResult, errors.New("Truncated response should have continuation token set") } + for i, obj := range listBucketResult.Contents { + listBucketResult.Contents[i].Key, err = url.QueryUnescape(obj.Key) + if err != nil { + return listBucketResult, err + } + } + + for i, obj := range listBucketResult.CommonPrefixes { + listBucketResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) + if err != nil { + return listBucketResult, err + } + } + // Success. return listBucketResult, nil } @@ -309,7 +360,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don var marker string for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 1000) + result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 0) if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -333,12 +384,9 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { - object := ObjectInfo{} - object.Key = obj.Prefix - object.Size = 0 select { // Send object prefixes. - case objectStatCh <- object: + case objectStatCh <- ObjectInfo{Key: obj.Prefix}: // If receives done from the caller, return here. case <-doneCh: return @@ -380,25 +428,25 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) - // Set object prefix. - if objectPrefix != "" { - urlValues.Set("prefix", objectPrefix) - } + + // Set object prefix, prefix value to be set to empty is okay. + urlValues.Set("prefix", objectPrefix) + + // Set delimiter, delimiter value to be set to empty is okay. + urlValues.Set("delimiter", delimiter) + // Set object marker. if objectMarker != "" { urlValues.Set("marker", objectMarker) } - // Set delimiter. - if delimiter != "" { - urlValues.Set("delimiter", delimiter) - } - // maxkeys should default to 1000 or less. - if maxkeys == 0 || maxkeys > 1000 { - maxkeys = 1000 - } // Set max keys. - urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + if maxkeys > 0 { + urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) + } + + // Always set encoding-type + urlValues.Set("encoding-type", "url") // Execute GET on bucket to list objects. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ @@ -421,6 +469,28 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit if err != nil { return listBucketResult, err } + + for i, obj := range listBucketResult.Contents { + listBucketResult.Contents[i].Key, err = url.QueryUnescape(obj.Key) + if err != nil { + return listBucketResult, err + } + } + + for i, obj := range listBucketResult.CommonPrefixes { + listBucketResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) + if err != nil { + return listBucketResult, err + } + } + + if listBucketResult.NextMarker != "" { + listBucketResult.NextMarker, err = url.QueryUnescape(listBucketResult.NextMarker) + if err != nil { + return listBucketResult, err + } + } + return listBucketResult, nil } @@ -484,16 +554,16 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive var uploadIDMarker string for { // list all multipart uploads. - result, err := c.listMultipartUploadsQuery(bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 1000) + result, err := c.listMultipartUploadsQuery(bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 0) if err != nil { objectMultipartStatCh <- ObjectMultipartInfo{ Err: err, } return } - // Save objectMarker and uploadIDMarker for next request. objectMarker = result.NextKeyMarker uploadIDMarker = result.NextUploadIDMarker + // Send all multipart uploads. for _, obj := range result.Uploads { // Calculate total size of the uploaded parts if 'aggregateSize' is enabled. @@ -518,12 +588,9 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { - object := ObjectMultipartInfo{} - object.Key = obj.Prefix - object.Size = 0 select { // Send delimited prefixes here. - case objectMultipartStatCh <- object: + case objectMultipartStatCh <- ObjectMultipartInfo{Key: obj.Prefix, Size: 0}: // If done channel return here. case <-doneCh: return @@ -563,21 +630,21 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, if uploadIDMarker != "" { urlValues.Set("upload-id-marker", uploadIDMarker) } - // Set prefix marker. - if prefix != "" { - urlValues.Set("prefix", prefix) - } - // Set delimiter. - if delimiter != "" { - urlValues.Set("delimiter", delimiter) - } + + // Set object prefix, prefix value to be set to empty is okay. + urlValues.Set("prefix", prefix) + + // Set delimiter, delimiter value to be set to empty is okay. + urlValues.Set("delimiter", delimiter) + + // Always set encoding-type + urlValues.Set("encoding-type", "url") // maxUploads should be 1000 or less. - if maxUploads == 0 || maxUploads > 1000 { - maxUploads = 1000 + if maxUploads > 0 { + // Set max-uploads. + urlValues.Set("max-uploads", fmt.Sprintf("%d", maxUploads)) } - // Set max-uploads. - urlValues.Set("max-uploads", fmt.Sprintf("%d", maxUploads)) // Execute GET on bucketName to list multipart uploads. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ @@ -600,6 +667,31 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, if err != nil { return listMultipartUploadsResult, err } + + listMultipartUploadsResult.NextKeyMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextKeyMarker) + if err != nil { + return listMultipartUploadsResult, err + } + + listMultipartUploadsResult.NextUploadIDMarker, err = url.QueryUnescape(listMultipartUploadsResult.NextUploadIDMarker) + if err != nil { + return listMultipartUploadsResult, err + } + + for i, obj := range listMultipartUploadsResult.Uploads { + listMultipartUploadsResult.Uploads[i].Key, err = url.QueryUnescape(obj.Key) + if err != nil { + return listMultipartUploadsResult, err + } + } + + for i, obj := range listMultipartUploadsResult.CommonPrefixes { + listMultipartUploadsResult.CommonPrefixes[i].Prefix, err = url.QueryUnescape(obj.Prefix) + if err != nil { + return listMultipartUploadsResult, err + } + } + return listMultipartUploadsResult, nil } @@ -688,11 +780,10 @@ func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, pa urlValues.Set("uploadId", uploadID) // maxParts should be 1000 or less. - if maxParts == 0 || maxParts > 1000 { - maxParts = 1000 + if maxParts > 0 { + // Set max parts. + urlValues.Set("max-parts", fmt.Sprintf("%d", maxParts)) } - // Set max parts. - urlValues.Set("max-parts", fmt.Sprintf("%d", maxParts)) // Execute GET on objectName to get list of parts. resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ |