For plusWorkaround, correctly handle cleanup, deletions. #239

s3/public.go

@@ -145,6 +145,11 @@ func (storage *PublishedStorage) Remove(path string) error {
 	if err != nil {
 		return fmt.Errorf("error deleting %s from %s: %s", path, storage, err)
 	}
+
+	if storage.plusWorkaround && strings.Index(path, "+") != -1 {
+		// try to remove workaround version, but don't care about result
+		_ = storage.Remove(strings.Replace(path, "+", " ", -1))
+	}
 	return nil
 }
 
@@ -152,7 +157,7 @@ func (storage *PublishedStorage) Remove(path string) error {
 func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
 	const page = 1000
 
-	filelist, err := storage.Filelist(path)
+	filelist, err := storage.internalFilelist(path, false)
 	if err != nil {
 		return err
 	}
@@ -233,6 +238,10 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 
 // Filelist returns list of files under prefix
 func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
+	return storage.internalFilelist(prefix, true)
+}
+
+func (storage *PublishedStorage) internalFilelist(prefix string, hidePlusWorkaround bool) ([]string, error) {
 	result := []string{}
 	marker := ""
 	prefix = filepath.Join(storage.prefix, prefix)
@@ -246,12 +255,18 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 		}
 		lastKey := ""
 		for _, key := range contents.Contents {
+			lastKey = key.Key
+			if storage.plusWorkaround && hidePlusWorkaround && strings.Index(lastKey, " ") != -1 {
+				// if we use plusWorkaround, we want to hide those duplicates
+				/// from listing
+				continue
+			}
+
 			if prefix == "" {
 				result = append(result, key.Key)
 			} else {
 				result = append(result, key.Key[len(prefix):])
 			}
-			lastKey = key.Key
 		}
 		if contents.IsTruncated {
 			marker = contents.NextMarker

s3/public_test.go

@@ -108,6 +108,33 @@ func (s *PublishedStorageSuite) TestFilelist(c *C) {
 	c.Check(list, DeepEquals, []string{"a", "b", "c"})
 }
 
+func (s *PublishedStorageSuite) TestFilelistPlusWorkaround(c *C) {
+	s.storage.plusWorkaround = true
+	s.prefixedStorage.plusWorkaround = true
+
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")
+		c.Check(err, IsNil)
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a+b", "lala/c", "test/a+1", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a+1"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a+b", "c"})
+}
+
 func (s *PublishedStorageSuite) TestRemove(c *C) {
 	err := s.storage.bucket.Put("a/b", []byte("test"), "binary/octet-stream", "private")
 	c.Check(err, IsNil)
@@ -119,7 +146,50 @@ func (s *PublishedStorageSuite) TestRemove(c *C) {
 	c.Check(err, ErrorMatches, "The specified key does not exist.")
 }
 
+func (s *PublishedStorageSuite) TestRemovePlusWorkaround(c *C) {
+	s.storage.plusWorkaround = true
+
+	err := s.storage.bucket.Put("a/b+c", []byte("test"), "binary/octet-stream", "private")
+	c.Check(err, IsNil)
+
+	err = s.storage.bucket.Put("a/b", []byte("test"), "binary/octet-stream", "private")
+	c.Check(err, IsNil)
+
+	err = s.storage.Remove("a/b+c")
+	c.Check(err, IsNil)
+
+	_, err = s.storage.bucket.Get("a/b+c")
+	c.Check(err, ErrorMatches, "The specified key does not exist.")
+
+	_, err = s.storage.bucket.Get("a/b c")
+	c.Check(err, ErrorMatches, "The specified key does not exist.")
+
+	err = s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	_, err = s.storage.bucket.Get("a/b")
+	c.Check(err, ErrorMatches, "The specified key does not exist.")
+
+}
+
 func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
+	s.storage.plusWorkaround = true
+
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")
+		c.Check(err, IsNil)
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a+b", "lala/c", "testa"})
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirsPlusWorkaround(c *C) {
 	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
 	for _, path := range paths {
 		err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")