From 38dfe3435a1d5ddd5909320c360f7b2ab592bf7c Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Fri, 29 May 2015 02:13:59 +0300 Subject: [PATCH] For plusWorkaround, correctly handle cleanup, deletions. #239 --- s3/public.go | 19 +++++++++++-- s3/public_test.go | 70 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/s3/public.go b/s3/public.go index 5d084e58..4aad2856 100644 --- a/s3/public.go +++ b/s3/public.go @@ -145,6 +145,11 @@ func (storage *PublishedStorage) Remove(path string) error { if err != nil { return fmt.Errorf("error deleting %s from %s: %s", path, storage, err) } + + if storage.plusWorkaround && strings.Index(path, "+") != -1 { + // try to remove workaround version, but don't care about result + _ = storage.Remove(strings.Replace(path, "+", " ", -1)) + } return nil } @@ -152,7 +157,7 @@ func (storage *PublishedStorage) Remove(path string) error { func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error { const page = 1000 - filelist, err := storage.Filelist(path) + filelist, err := storage.internalFilelist(path, false) if err != nil { return err } @@ -233,6 +238,10 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP // Filelist returns list of files under prefix func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) { + return storage.internalFilelist(prefix, true) +} + +func (storage *PublishedStorage) internalFilelist(prefix string, hidePlusWorkaround bool) ([]string, error) { result := []string{} marker := "" prefix = filepath.Join(storage.prefix, prefix) @@ -246,12 +255,18 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) { } lastKey := "" for _, key := range contents.Contents { + lastKey = key.Key + if storage.plusWorkaround && hidePlusWorkaround && strings.Index(lastKey, " ") != -1 { + // if we use plusWorkaround, we want to hide those duplicates + /// from listing + continue + } + if prefix == "" { result = append(result, key.Key) } else { result = append(result, key.Key[len(prefix):]) } - lastKey = key.Key } if contents.IsTruncated { marker = contents.NextMarker diff --git a/s3/public_test.go b/s3/public_test.go index 0b25bb47..f19945b8 100644 --- a/s3/public_test.go +++ b/s3/public_test.go @@ -108,6 +108,33 @@ func (s *PublishedStorageSuite) TestFilelist(c *C) { c.Check(list, DeepEquals, []string{"a", "b", "c"}) } +func (s *PublishedStorageSuite) TestFilelistPlusWorkaround(c *C) { + s.storage.plusWorkaround = true + s.prefixedStorage.plusWorkaround = true + + paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"} + for _, path := range paths { + err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private") + c.Check(err, IsNil) + } + + list, err := s.storage.Filelist("") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a+b", "lala/c", "test/a+1", "testa"}) + + list, err = s.storage.Filelist("test") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"a+1"}) + + list, err = s.storage.Filelist("test2") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{}) + + list, err = s.prefixedStorage.Filelist("") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"a+b", "c"}) +} + func (s *PublishedStorageSuite) TestRemove(c *C) { err := s.storage.bucket.Put("a/b", []byte("test"), "binary/octet-stream", "private") c.Check(err, IsNil) @@ -119,7 +146,50 @@ func (s *PublishedStorageSuite) TestRemove(c *C) { c.Check(err, ErrorMatches, "The specified key does not exist.") } +func (s *PublishedStorageSuite) TestRemovePlusWorkaround(c *C) { + s.storage.plusWorkaround = true + + err := s.storage.bucket.Put("a/b+c", []byte("test"), "binary/octet-stream", "private") + c.Check(err, IsNil) + + err = s.storage.bucket.Put("a/b", []byte("test"), "binary/octet-stream", "private") + c.Check(err, IsNil) + + err = s.storage.Remove("a/b+c") + c.Check(err, IsNil) + + _, err = s.storage.bucket.Get("a/b+c") + c.Check(err, ErrorMatches, "The specified key does not exist.") + + _, err = s.storage.bucket.Get("a/b c") + c.Check(err, ErrorMatches, "The specified key does not exist.") + + err = s.storage.Remove("a/b") + c.Check(err, IsNil) + + _, err = s.storage.bucket.Get("a/b") + c.Check(err, ErrorMatches, "The specified key does not exist.") + +} + func (s *PublishedStorageSuite) TestRemoveDirs(c *C) { + s.storage.plusWorkaround = true + + paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"} + for _, path := range paths { + err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private") + c.Check(err, IsNil) + } + + err := s.storage.RemoveDirs("test", nil) + c.Check(err, IsNil) + + list, err := s.storage.Filelist("") + c.Check(err, IsNil) + c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a+b", "lala/c", "testa"}) +} + +func (s *PublishedStorageSuite) TestRemoveDirsPlusWorkaround(c *C) { paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"} for _, path := range paths { err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")