package s3 import ( "bytes" "io/ioutil" "os" "path/filepath" . "gopkg.in/check.v1" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/smira/aptly/files" ) type PublishedStorageSuite struct { srv *Server storage, prefixedStorage *PublishedStorage } var _ = Suite(&PublishedStorageSuite{}) func (s *PublishedStorageSuite) SetUpTest(c *C) { var err error s.srv, err = NewServer(&Config{}) c.Assert(err, IsNil) c.Assert(s.srv, NotNil) s.storage, err = NewPublishedStorage("aa", "bb", "", "test-1", s.srv.URL(), "test", "", "", "", "", false, true) c.Assert(err, IsNil) s.prefixedStorage, err = NewPublishedStorage("aa", "bb", "", "test-1", s.srv.URL(), "test", "", "lala", "", "", false, true) c.Assert(err, IsNil) _, err = s.storage.s3.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String("test")}) c.Assert(err, IsNil) } func (s *PublishedStorageSuite) TearDownTest(c *C) { s.srv.Quit() } func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte { resp, err := s.storage.s3.GetObject(&s3.GetObjectInput{ Bucket: aws.String(s.storage.bucket), Key: aws.String(path), }) c.Assert(err, IsNil) body, err := ioutil.ReadAll(resp.Body) resp.Body.Close() c.Assert(err, IsNil) return body } func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) { _, err := s.storage.s3.HeadObject(&s3.HeadObjectInput{ Bucket: aws.String(s.storage.bucket), Key: aws.String(path), }) c.Assert(err, ErrorMatches, ".*\n.*status code: 404.*") } func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) { _, err := s.storage.s3.PutObject(&s3.PutObjectInput{ Bucket: aws.String(s.storage.bucket), Key: aws.String(path), Body: bytes.NewReader(data), ContentType: aws.String("binary/octet-stream"), ACL: aws.String("private"), }) c.Assert(err, IsNil) } func (s *PublishedStorageSuite) TestPutFile(c *C) { dir := c.MkDir() err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to s3!"), 0644) c.Assert(err, IsNil) err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a")) c.Check(err, IsNil) c.Check(s.GetFile(c, "a/b.txt"), DeepEquals, []byte("welcome to s3!")) err = s.prefixedStorage.PutFile("a/b.txt", filepath.Join(dir, "a")) c.Check(err, IsNil) c.Check(s.GetFile(c, "lala/a/b.txt"), DeepEquals, []byte("welcome to s3!")) } func (s *PublishedStorageSuite) TestPutFilePlusWorkaround(c *C) { s.storage.plusWorkaround = true dir := c.MkDir() err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to s3!"), 0644) c.Assert(err, IsNil) err = s.storage.PutFile("a/b+c.txt", filepath.Join(dir, "a")) c.Check(err, IsNil) c.Check(s.GetFile(c, "a/b+c.txt"), DeepEquals, []byte("welcome to s3!")) c.Check(s.GetFile(c, "a/b c.txt"), DeepEquals, []byte("welcome to s3!")) } func (s *PublishedStorageSuite) TestFilelist(c *C) { paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"} for _, path := range paths { s.PutFile(c, path, []byte("test")) } list, err := s.storage.Filelist("") c.Check(err, IsNil) c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"}) list, err = s.storage.Filelist("test") c.Check(err, IsNil) c.Check(list, DeepEquals, []string{"a", "b"}) list, err = s.storage.Filelist("test2") c.Check(err, IsNil) c.Check(list, DeepEquals, []string{}) list, err = s.prefixedStorage.Filelist("") c.Check(err, IsNil) c.Check(list, DeepEquals, []string{"a", "b", "c"}) } func (s *PublishedStorageSuite) TestFilelistPlusWorkaround(c *C) { s.storage.plusWorkaround = true s.prefixedStorage.plusWorkaround = true paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"} for _, path := range paths { s.PutFile(c, path, []byte("test")) } list, err := s.storage.Filelist("") c.Check(err, IsNil) c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a+b", "lala/c", "test/a+1", "testa"}) list, err = s.storage.Filelist("test") c.Check(err, IsNil) c.Check(list, DeepEquals, []string{"a+1"}) list, err = s.storage.Filelist("test2") c.Check(err, IsNil) c.Check(list, DeepEquals, []string{}) list, err = s.prefixedStorage.Filelist("") c.Check(err, IsNil) c.Check(list, DeepEquals, []string{"a+b", "c"}) } func (s *PublishedStorageSuite) TestRemove(c *C) { s.PutFile(c, "a/b", []byte("test")) err := s.storage.Remove("a/b") c.Check(err, IsNil) s.AssertNoFile(c, "a/b") } func (s *PublishedStorageSuite) TestRemovePlusWorkaround(c *C) { s.storage.plusWorkaround = true s.PutFile(c, "a/b+c", []byte("test")) s.PutFile(c, "a/b", []byte("test")) err := s.storage.Remove("a/b+c") c.Check(err, IsNil) s.AssertNoFile(c, "a/b+c") s.AssertNoFile(c, "a/b c") err = s.storage.Remove("a/b") c.Check(err, IsNil) s.AssertNoFile(c, "a/b") } func (s *PublishedStorageSuite) TestRemoveDirs(c *C) { s.storage.plusWorkaround = true paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"} for _, path := range paths { s.PutFile(c, path, []byte("test")) } err := s.storage.RemoveDirs("test", nil) c.Check(err, IsNil) list, err := s.storage.Filelist("") c.Check(err, IsNil) c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a+b", "lala/c", "testa"}) } func (s *PublishedStorageSuite) TestRemoveDirsPlusWorkaround(c *C) { paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"} for _, path := range paths { s.PutFile(c, path, []byte("test")) } err := s.storage.RemoveDirs("test", nil) c.Check(err, IsNil) list, err := s.storage.Filelist("") c.Check(err, IsNil) c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "testa"}) } func (s *PublishedStorageSuite) TestRenameFile(c *C) { c.Skip("copy not available in s3test") } func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { root := c.MkDir() pool := files.NewPackagePool(root) sourcePath := filepath.Join(root, "pool/c1/df/mars-invaders_1.03.deb") err := os.MkdirAll(filepath.Dir(sourcePath), 0755) c.Assert(err, IsNil) err = ioutil.WriteFile(sourcePath, []byte("Contents"), 0644) c.Assert(err, IsNil) sourcePath2 := filepath.Join(root, "pool/e9/df/mars-invaders_1.03.deb") err = os.MkdirAll(filepath.Dir(sourcePath2), 0755) c.Assert(err, IsNil) err = ioutil.WriteFile(sourcePath2, []byte("Spam"), 0644) c.Assert(err, IsNil) // first link from pool err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "c1df1da7a1ce305a3b60af9d5733ac1d", false) c.Check(err, IsNil) c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) // duplicate link from pool err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "c1df1da7a1ce305a3b60af9d5733ac1d", false) c.Check(err, IsNil) c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) // link from pool with conflict err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath2, "e9dfd31cc505d51fc26975250750deab", false) c.Check(err, ErrorMatches, ".*file already exists and is different.*") c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) // link from pool with conflict and force err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath2, "e9dfd31cc505d51fc26975250750deab", true) c.Check(err, IsNil) c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam")) }