Support for custom endpoints and multi del disabling. #218

s3/public.go

@@ -20,6 +20,7 @@ type PublishedStorage struct {
 	storageClass     string
 	encryptionMethod string
 	plusWorkaround   bool
+	disableMultiDel  bool
 }
 
 // Check interface
@@ -29,7 +30,7 @@ var (
 
 // NewPublishedStorageRaw creates published storage from raw aws credentials
 func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL, prefix,
-	storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
+	storageClass, encryptionMethod string, plusWorkaround, disabledMultiDel bool) (*PublishedStorage, error) {
 	if defaultACL == "" {
 		defaultACL = "private"
 	}
@@ -44,7 +45,8 @@ func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL
 		prefix:           prefix,
 		storageClass:     storageClass,
 		encryptionMethod: encryptionMethod,
-		plusWorkaround:   plusWorkaround}
+		plusWorkaround:   plusWorkaround,
+		disableMultiDel:  disabledMultiDel}
 	result.bucket = result.s3.Bucket(bucket)
 
 	return result, nil
@@ -52,19 +54,33 @@ func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL
 
 // NewPublishedStorage creates new instance of PublishedStorage with specified S3 access
 // keys, region and bucket name
-func NewPublishedStorage(accessKey, secretKey, region, bucket, defaultACL, prefix,
+func NewPublishedStorage(accessKey, secretKey, region, endpoint, bucket, defaultACL, prefix,
-	storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
+	storageClass, encryptionMethod string, plusWorkaround, disableMultiDel bool) (*PublishedStorage, error) {
 	auth, err := aws.GetAuth(accessKey, secretKey)
 	if err != nil {
 		return nil, err
 	}
 
-	awsRegion, ok := aws.Regions[region]
+	var awsRegion aws.Region
-	if !ok {
+
-		return nil, fmt.Errorf("unknown region: %#v", region)
+	if endpoint == "" {
+		var ok bool
+
+		awsRegion, ok = aws.Regions[region]
+		if !ok {
+			return nil, fmt.Errorf("unknown region: %#v", region)
+		}
+	} else {
+		awsRegion = aws.Region{
+			Name:                 region,
+			S3Endpoint:           endpoint,
+			S3LocationConstraint: true,
+			S3LowercaseBucket:    true,
+		}
 	}
 
-	return NewPublishedStorageRaw(auth, awsRegion, bucket, defaultACL, prefix, storageClass, encryptionMethod, plusWorkaround)
+	return NewPublishedStorageRaw(auth, awsRegion, bucket, defaultACL, prefix, storageClass, encryptionMethod,
+		plusWorkaround, disableMultiDel)
 }
 
 // String
@@ -135,27 +151,33 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 		return err
 	}
 
-	numParts := (len(filelist) + page - 1) / page
+	if storage.disableMultiDel {
-
+		for i := range filelist {
-	for i := 0; i < numParts; i++ {
+			err = storage.bucket.Del(filepath.Join(storage.prefix, path, filelist[i]))
-		var part []string
+			if err != nil {
-		if i == numParts-1 {
+				return fmt.Errorf("error deleting path %s from %s: %s", filelist[i], storage, err)
-			part = filelist[i*page:]
+			}
-		} else {
-			part = filelist[i*page : (i+1)*page]
 		}
-		paths := make([]string, len(part))
+	} else {
+		numParts := (len(filelist) + page - 1) / page
 
-		for i := range part {
+		for i := 0; i < numParts; i++ {
-			paths[i] = filepath.Join(storage.prefix, path, part[i])
+			var part []string
-		}
+			if i == numParts-1 {
+				part = filelist[i*page:]
+			} else {
+				part = filelist[i*page : (i+1)*page]
+			}
+			paths := make([]string, len(part))
 
-		err = storage.bucket.MultiDel(paths)
+			for i := range part {
-		if err != nil {
+				paths[i] = filepath.Join(storage.prefix, path, part[i])
-			return fmt.Errorf("error deleting multiple paths from %s: %s", storage, err)
+			}
-		}
+
-		if err != nil {
+			err = storage.bucket.MultiDel(paths)
-			return err
+			if err != nil {
+				return fmt.Errorf("error deleting multiple paths from %s: %s", storage, err)
+			}
 		}
 	}
 

s3/public_test.go

@@ -25,10 +25,10 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 	c.Assert(s.srv, NotNil)
 
 	auth, _ := aws.GetAuth("aa", "bb")
-	s.storage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "", "", "", false)
+	s.storage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "", "", "", false, true)
 	c.Assert(err, IsNil)
 
-	s.prefixedStorage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "lala", "", "", false)
+	s.prefixedStorage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "lala", "", "", false, true)
 	c.Assert(err, IsNil)
 
 	err = s.storage.s3.Bucket("test").PutBucket("private")
@@ -40,7 +40,7 @@ func (s *PublishedStorageSuite) TearDownTest(c *C) {
 }
 
 func (s *PublishedStorageSuite) TestNewPublishedStorage(c *C) {
-	stor, err := NewPublishedStorage("aa", "bbb", "", "", "", "", "", "", false)
+	stor, err := NewPublishedStorage("aa", "bbb", "", "", "", "", "", "", "", false, false)
 	c.Check(stor, IsNil)
 	c.Check(err, ErrorMatches, "unknown region: .*")
 }
@@ -120,8 +120,6 @@ func (s *PublishedStorageSuite) TestRemove(c *C) {
 }
 
 func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
-	c.Skip("multiple-delete not available in s3test")
-
 	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
 	for _, path := range paths {
 		err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")
@@ -133,8 +131,7 @@ func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
 
 	list, err := s.storage.Filelist("")
 	c.Check(err, IsNil)
-	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "testa"})
-
 }
 
 func (s *PublishedStorageSuite) TestRenameFile(c *C) {