vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-05-07 22:20:24 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #371 from smira/s3-list-fix

Browse Source 
Replace object listing with SDK-standard iteration.
This commit is contained in:
Andrey Smirnov
2016-03-27 23:01:53 +03:00
parent 165a1c53b5 d9a1299f6b
commit 90ea1111e2
1 changed files with 14 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
s3/public.go
+14 -29
View File
@@ -287,26 +287,20 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
func (storage *PublishedStorage) internalFilelist(prefix string, hidePlusWorkaround bool) (paths []string, md5s []string, err error) { func (storage *PublishedStorage) internalFilelist(prefix string, hidePlusWorkaround bool) (paths []string, md5s []string, err error) {
paths = make([]string, 0, 1024) paths = make([]string, 0, 1024)
md5s = make([]string, 0, 1024) md5s = make([]string, 0, 1024)
marker := ""
prefix = filepath.Join(storage.prefix, prefix) prefix = filepath.Join(storage.prefix, prefix)
if prefix != "" { if prefix != "" {
prefix += "/" prefix += "/"
} }
for {
params := &s3.ListObjectsInput{
Bucket: aws.String(storage.bucket),
Prefix: aws.String(prefix),
MaxKeys: aws.Int64(1000),
}
contents, err := storage.s3.ListObjects(params) params := &s3.ListObjectsInput{
if err != nil { Bucket: aws.String(storage.bucket),
return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err) Prefix: aws.String(prefix),
} MaxKeys: aws.Int64(1000),
lastKey := "" }
err = storage.s3.ListObjectsPages(params, func(contents *s3.ListObjectsOutput, lastPage bool) bool {
for _, key := range contents.Contents { for _, key := range contents.Contents {
lastKey = *key.Key if storage.plusWorkaround && hidePlusWorkaround && strings.Index(*key.Key, " ") != -1 {
if storage.plusWorkaround && hidePlusWorkaround && strings.Index(lastKey, " ") != -1 {
// if we use plusWorkaround, we want to hide those duplicates // if we use plusWorkaround, we want to hide those duplicates
/// from listing /// from listing
continue continue
@@ -319,21 +313,12 @@ func (storage *PublishedStorage) internalFilelist(prefix string, hidePlusWorkaro
} }
md5s = append(md5s, strings.Replace(*key.ETag, "\"", "", -1)) md5s = append(md5s, strings.Replace(*key.ETag, "\"", "", -1))
} }
if contents.IsTruncated != nil && *contents.IsTruncated {
marker = "" return true
if contents.NextMarker != nil { })
marker = *contents.NextMarker
} if err != nil {
if marker == "" { return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
// From the s3 docs: If response does not include the
// NextMarker and it is truncated, you can use the value of the
// last Key in the response as the marker in the subsequent
// request to get the next set of object keys.
marker = lastKey
}
} else {
break
}
} }
return paths, md5s, nil return paths, md5s, nil