vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-06-04 05:10:40 +00:00
Code Issues Projects Releases Wiki Activity

Support for custom endpoints and multi del disabling. #218

Browse Source
This commit is contained in:
Andrey Smirnov
2015-04-17 01:16:50 +03:00
parent ba08ffe38b
commit 0791c88a02
2 changed files with 52 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
s3/public.go
+48 -26
View File
@@ -20,6 +20,7 @@ type PublishedStorage struct {
storageClass string
encryptionMethod string
plusWorkaround bool
disableMultiDel bool
}
// Check interface
@@ -29,7 +30,7 @@ var (
// NewPublishedStorageRaw creates published storage from raw aws credentials
func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL, prefix,
storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
storageClass, encryptionMethod string, plusWorkaround, disabledMultiDel bool) (*PublishedStorage, error) {
if defaultACL == "" {
defaultACL = "private"
}
@@ -44,7 +45,8 @@ func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL
prefix: prefix,
storageClass: storageClass,
encryptionMethod: encryptionMethod,
plusWorkaround: plusWorkaround}
plusWorkaround: plusWorkaround,
disableMultiDel: disabledMultiDel}
result.bucket = result.s3.Bucket(bucket)
return result, nil
@@ -52,19 +54,33 @@ func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL
// NewPublishedStorage creates new instance of PublishedStorage with specified S3 access
// keys, region and bucket name
func NewPublishedStorage(accessKey, secretKey, region, bucket, defaultACL, prefix,
storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
func NewPublishedStorage(accessKey, secretKey, region, endpoint, bucket, defaultACL, prefix,
storageClass, encryptionMethod string, plusWorkaround, disableMultiDel bool) (*PublishedStorage, error) {
auth, err := aws.GetAuth(accessKey, secretKey)
if err != nil {
return nil, err
}
awsRegion, ok := aws.Regions[region]
if !ok {
return nil, fmt.Errorf("unknown region: %#v", region)
var awsRegion aws.Region
if endpoint == "" {
var ok bool
awsRegion, ok = aws.Regions[region]
if !ok {
return nil, fmt.Errorf("unknown region: %#v", region)
}
} else {
awsRegion = aws.Region{
Name: region,
S3Endpoint: endpoint,
S3LocationConstraint: true,
S3LowercaseBucket: true,
}
}
return NewPublishedStorageRaw(auth, awsRegion, bucket, defaultACL, prefix, storageClass, encryptionMethod, plusWorkaround)
return NewPublishedStorageRaw(auth, awsRegion, bucket, defaultACL, prefix, storageClass, encryptionMethod,
plusWorkaround, disableMultiDel)
}
// String
@@ -135,27 +151,33 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
return err
}
numParts := (len(filelist) + page - 1) / page
for i := 0; i < numParts; i++ {
var part []string
if i == numParts-1 {
part = filelist[i*page:]
} else {
part = filelist[i*page : (i+1)*page]
if storage.disableMultiDel {
for i := range filelist {
err = storage.bucket.Del(filepath.Join(storage.prefix, path, filelist[i]))
if err != nil {
return fmt.Errorf("error deleting path %s from %s: %s", filelist[i], storage, err)
}
}
paths := make([]string, len(part))
} else {
numParts := (len(filelist) + page - 1) / page
for i := range part {
paths[i] = filepath.Join(storage.prefix, path, part[i])
}
for i := 0; i < numParts; i++ {
var part []string
if i == numParts-1 {
part = filelist[i*page:]
} else {
part = filelist[i*page : (i+1)*page]
}
paths := make([]string, len(part))
err = storage.bucket.MultiDel(paths)
if err != nil {
return fmt.Errorf("error deleting multiple paths from %s: %s", storage, err)
}
if err != nil {
return err
for i := range part {
paths[i] = filepath.Join(storage.prefix, path, part[i])
}
err = storage.bucket.MultiDel(paths)
if err != nil {
return fmt.Errorf("error deleting multiple paths from %s: %s", storage, err)
}
}
}
s3/public_test.go
+4 -7
View File
@@ -25,10 +25,10 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
c.Assert(s.srv, NotNil)
auth, _ := aws.GetAuth("aa", "bb")
s.storage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "", "", "", false)
s.storage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "", "", "", false, true)
c.Assert(err, IsNil)
s.prefixedStorage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "lala", "", "", false)
s.prefixedStorage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "lala", "", "", false, true)
c.Assert(err, IsNil)
err = s.storage.s3.Bucket("test").PutBucket("private")
@@ -40,7 +40,7 @@ func (s *PublishedStorageSuite) TearDownTest(c *C) {
}
func (s *PublishedStorageSuite) TestNewPublishedStorage(c *C) {
stor, err := NewPublishedStorage("aa", "bbb", "", "", "", "", "", "", false)
stor, err := NewPublishedStorage("aa", "bbb", "", "", "", "", "", "", "", false, false)
c.Check(stor, IsNil)
c.Check(err, ErrorMatches, "unknown region: .*")
}
@@ -120,8 +120,6 @@ func (s *PublishedStorageSuite) TestRemove(c *C) {
}
func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
c.Skip("multiple-delete not available in s3test")
paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
for _, path := range paths {
err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")
@@ -133,8 +131,7 @@ func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
list, err := s.storage.Filelist("")
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "testa"})
}
func (s *PublishedStorageSuite) TestRenameFile(c *C) {