Introduce plusWorkaround: generate copy of file with '+' -> ' ' to workaround S3/apt bug. #98

cmd/context.go

@@ -217,7 +217,8 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 
 			var err error
 			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
-				params.Region, params.Bucket, params.ACL, params.Prefix, params.StorageClass, params.EncryptionMethod)
+				params.Region, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
+				params.EncryptionMethod, params.PlusWorkaround)
 			if err != nil {
 				Fatal(err)
 			}

man/aptly.1

@@ -57,7 +57,8 @@ Configuration file is stored in JSON format (default values shown below):
       "prefix": "",
       "acl": "public\-read",
       "storageClass": "",
-      "encryptionMethod": ""
+      "encryptionMethod": "",
+      "plusWorkaround": false
     }
 }
 .
@@ -117,7 +118,7 @@ if enabled, all mirrors created would have flag set to download source packages;
 specifies paramaters for short PPA url expansion, if left blank they default to output of \fBlsb_release\fR command
 .
 .TP
-\fBS3PublisEndpoints\fR
+\fBS3PublishEndpoints\fR
 configuration of Amazon S3 publishing endpoints (see below)
 .
 .SH "S3 PUBLISHING ENDPOINTS"
@@ -151,6 +152,10 @@ bucket name
 \fBencryptionMethod\fR
 (optional) server\-side encryption method, defaults to none\. Currently the only available encryption method is \fBAES256\fR
 .
+.TP
+\fBplusWorkaround\fR
+(optional) workaround misbehavior in apt and Amazon S3 for files with \fB+\fR in filename by creating two copies of package files with \fB+\fR in filename: one original and another one with spaces instead of plus signs With \fBplusWorkaround\fR enabled, package files with plus sign would be stored twice\. aptly might not cleanup files with spaces when published repository is dropped or updated (switched) to new version of repository (snapshot)\.
+.
 .P
 In order to publish to S3, specify endpoint as \fBs3:endpoint\-name:\fR before publishing prefix on the command line, e\.g\.:
 .

man/aptly.1.ronn.tmpl

@@ -49,7 +49,8 @@ Configuration file is stored in JSON format (default values shown below):
           "prefix": "",
           "acl": "public-read",
           "storageClass": "",
-          "encryptionMethod": ""
+          "encryptionMethod": "",
+          "plusWorkaround": false
         }
     }
 
@@ -97,7 +98,7 @@ Options:
     specifies paramaters for short PPA url expansion, if left blank they default
     to output of `lsb_release` command
 
-  * `S3PublisEndpoints`:
+  * `S3PublishEndpoints`:
     configuration of Amazon S3 publishing endpoints (see below)
 
 ## S3 PUBLISHING ENDPOINTS
@@ -129,7 +130,14 @@ and associated settings:
    * `encryptionMethod`:
      (optional) server-side encryption method, defaults to none. Currently
      the only available encryption method is `AES256`
-
+   * `plusWorkaround`:
+     (optional) workaround misbehavior in apt and Amazon S3
+     for files with `+` in filename by
+     creating two copies of package files with `+` in filename: one original
+     and another one with spaces instead of plus signs
+     With `plusWorkaround` enabled, package files with plus sign
+     would be stored twice. aptly might not cleanup files with spaces when published
+     repository is dropped or updated (switched) to new version of repository (snapshot).
 
 In order to publish to S3, specify endpoint as `s3:endpoint-name:` before
 publishing prefix on the command line, e.g.:

s3/public.go

@@ -19,6 +19,7 @@ type PublishedStorage struct {
 	prefix           string
 	storageClass     string
 	encryptionMethod string
+	plusWorkaround   bool
 }
 
 // Check interface
@@ -28,7 +29,7 @@ var (
 
 // NewPublishedStorageRaw creates published storage from raw aws credentials
 func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL, prefix,
-	storageClass, encryptionMethod string) (*PublishedStorage, error) {
+	storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
 	if defaultACL == "" {
 		defaultACL = "private"
 	}
@@ -42,7 +43,8 @@ func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL
 		acl:              s3.ACL(defaultACL),
 		prefix:           prefix,
 		storageClass:     storageClass,
-		encryptionMethod: encryptionMethod}
+		encryptionMethod: encryptionMethod,
+		plusWorkaround:   plusWorkaround}
 	result.bucket = result.s3.Bucket(bucket)
 
 	return result, nil
@@ -51,7 +53,7 @@ func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL
 // NewPublishedStorage creates new instance of PublishedStorage with specified S3 access
 // keys, region and bucket name
 func NewPublishedStorage(accessKey, secretKey, region, bucket, defaultACL, prefix,
-	storageClass, encryptionMethod string) (*PublishedStorage, error) {
+	storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
 	auth, err := aws.GetAuth(accessKey, secretKey)
 	if err != nil {
 		return nil, err
@@ -62,7 +64,7 @@ func NewPublishedStorage(accessKey, secretKey, region, bucket, defaultACL, prefi
 		return nil, fmt.Errorf("unknown region: %#v", region)
 	}
 
-	return NewPublishedStorageRaw(auth, awsRegion, bucket, defaultACL, prefix, storageClass, encryptionMethod)
+	return NewPublishedStorageRaw(auth, awsRegion, bucket, defaultACL, prefix, storageClass, encryptionMethod, plusWorkaround)
 }
 
 // String
@@ -108,6 +110,10 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	if err != nil {
 		return fmt.Errorf("error uploading %s to %s: %s", sourceFilename, storage, err)
 	}
+
+	if storage.plusWorkaround && strings.Index(path, "+") != -1 {
+		return storage.PutFile(strings.Replace(path, "+", " ", -1), sourceFilename)
+	}
 	return nil
 }
 

s3/public_test.go

@@ -24,10 +24,10 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 	c.Assert(s.srv, NotNil)
 
 	auth, _ := aws.GetAuth("aa", "bb")
-	s.storage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "", "", "")
+	s.storage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "", "", "", false)
 	c.Assert(err, IsNil)
 
-	s.prefixedStorage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "lala", "", "")
+	s.prefixedStorage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "lala", "", "", false)
 	c.Assert(err, IsNil)
 
 	err = s.storage.s3.Bucket("test").PutBucket("private")
@@ -39,7 +39,7 @@ func (s *PublishedStorageSuite) TearDownTest(c *C) {
 }
 
 func (s *PublishedStorageSuite) TestNewPublishedStorage(c *C) {
-	stor, err := NewPublishedStorage("aa", "bbb", "", "", "", "", "", "")
+	stor, err := NewPublishedStorage("aa", "bbb", "", "", "", "", "", "", false)
 	c.Check(stor, IsNil)
 	c.Check(err, ErrorMatches, "unknown region: .*")
 }
@@ -64,6 +64,25 @@ func (s *PublishedStorageSuite) TestPutFile(c *C) {
 	c.Check(data, DeepEquals, []byte("welcome to s3!"))
 }
 
+func (s *PublishedStorageSuite) TestPutFilePlusWorkaround(c *C) {
+	s.storage.plusWorkaround = true
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to s3!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("a/b+c.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	data, err := s.storage.bucket.Get("a/b+c.txt")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+
+	data, err = s.storage.bucket.Get("a/b c.txt")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+}
+
 func (s *PublishedStorageSuite) TestFilelist(c *C) {
 	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
 	for _, path := range paths {

utils/config.go

@@ -34,6 +34,7 @@ type S3PublishRoot struct {
 	ACL              string `json:"acl"`
 	StorageClass     string `json:"storageClass"`
 	EncryptionMethod string `json:"encryptionMethod"`
+	PlusWorkaround   bool   `json:"plusWorkaround"`
 }
 
 // Config is configuration for aptly, shared by all modules

utils/config_test.go

@@ -67,7 +67,9 @@ func (s *ConfigSuite) TestSaveConfig(c *C) {
 		"      \"prefix\": \"\",\n"+
 		"      \"acl\": \"\",\n"+
 		"      \"storageClass\": \"\",\n"+
-		"      \"encryptionMethod\": \"\"\n"+"    }\n"+
+		"      \"encryptionMethod\": \"\",\n"+
+		"      \"plusWorkaround\": false\n"+
+		"    }\n"+
 		"  }\n"+
 		"}")
 }