diff --git a/aptly/interfaces.go b/aptly/interfaces.go index 9f28ff31..9980d898 100644 --- a/aptly/interfaces.go +++ b/aptly/interfaces.go @@ -70,7 +70,7 @@ type PublishedStorage interface { // Remove removes single file under public path Remove(path string) error // LinkFromPool links package file from pool to dist's pool location - LinkFromPool(prefix string, path string, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error + LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error // Filelist returns list of files under prefix Filelist(prefix string) ([]string, error) // RenameFile renames (moves) file diff --git a/azure/public.go b/azure/public.go index 1d210f77..7236cce4 100644 --- a/azure/public.go +++ b/azure/public.go @@ -22,7 +22,7 @@ import ( type PublishedStorage struct { container azblob.ContainerURL prefix string - pathCache map[string]string + pathCache map[string]map[string]string } // Check interface @@ -174,32 +174,38 @@ func (storage *PublishedStorage) Remove(path string) error { // LinkFromPool links package file from pool to dist's pool location // -// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/) +// publishedPrefix is desired prefix for the location in the pool. +// publishedRelParh is desired location in pool (like pool/component/liba/libav/) // sourcePool is instance of aptly.PackagePool // sourcePath is filepath to package file in package pool // // LinkFromPool returns relative path for the published file to be included in package index -func (storage *PublishedStorage) LinkFromPool(prefix string, path string, fileName string, sourcePool aptly.PackagePool, +func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { - publishedDirectory := filepath.Join(prefix, path) - relPath := filepath.Join(publishedDirectory, fileName) - poolPath := filepath.Join(storage.prefix, relPath) + relFilePath := filepath.Join(publishedRelPath, fileName) + prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath) + poolPath := filepath.Join(storage.prefix, prefixRelFilePath) if storage.pathCache == nil { - paths, md5s, err := storage.internalFilelist("") + storage.pathCache = make(map[string]map[string]string) + } + pathCache := storage.pathCache[publishedPrefix] + if pathCache == nil { + paths, md5s, err := storage.internalFilelist(publishedPrefix) if err != nil { return fmt.Errorf("error caching paths under prefix: %s", err) } - storage.pathCache = make(map[string]string, len(paths)) + pathCache = make(map[string]string, len(paths)) for i := range paths { - storage.pathCache[paths[i]] = md5s[i] + pathCache[paths[i]] = md5s[i] } + storage.pathCache[publishedPrefix] = pathCache } - destinationMD5, exists := storage.pathCache[relPath] + destinationMD5, exists := pathCache[relFilePath] sourceMD5 := sourceChecksums.MD5 if exists { @@ -222,9 +228,9 @@ func (storage *PublishedStorage) LinkFromPool(prefix string, path string, fileNa } defer source.Close() - err = storage.putFile(relPath, source, sourceMD5) + err = storage.putFile(prefixRelFilePath, source, sourceMD5) if err == nil { - storage.pathCache[relPath] = sourceMD5 + pathCache[relFilePath] = sourceMD5 } else { err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath)) } diff --git a/azure/public_test.go b/azure/public_test.go index 79abfed2..c038ba8b 100644 --- a/azure/public_test.go +++ b/azure/public_test.go @@ -300,45 +300,45 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Assert(err, IsNil) // first link from pool - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + err = s.storage.LinkFromPool("", filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) c.Check(err, IsNil) c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) // duplicate link from pool - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) c.Check(err, IsNil) c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) // link from pool with conflict - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false) + err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false) c.Check(err, ErrorMatches, ".*file already exists and is different.*") c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) // link from pool with conflict and force - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true) + err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true) c.Check(err, IsNil) c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam")) // for prefixed storage: // first link from pool - err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) c.Check(err, IsNil) // 2nd link from pool, providing wrong path for source file // // this test should check that file already exists in S3 and skip upload (which would fail if not skipped) s.prefixedStorage.pathCache = nil - err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false) + err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false) c.Check(err, IsNil) c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) // link from pool with nested file name - err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false) + err = s.storage.LinkFromPool("", "dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false) c.Check(err, IsNil) c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents")) diff --git a/files/public.go b/files/public.go index 93023819..7e9e3003 100644 --- a/files/public.go +++ b/files/public.go @@ -118,17 +118,17 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress // LinkFromPool links package file from pool to dist's pool location // -// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/) +// publishedPrefix is desired prefix for the location in the pool. +// publishedRelParh is desired location in pool (like pool/component/liba/libav/) // sourcePool is instance of aptly.PackagePool // sourcePath is a relative path to package file in package pool // // LinkFromPool returns relative path for the published file to be included in package index -func (storage *PublishedStorage) LinkFromPool(prefix string, path string, fileName string, sourcePool aptly.PackagePool, +func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { - publishedDirectory := filepath.Join(prefix, path) baseName := filepath.Base(fileName) - poolPath := filepath.Join(storage.rootPath, publishedDirectory, filepath.Dir(fileName)) + poolPath := filepath.Join(storage.rootPath, publishedPrefix, publishedRelPath, filepath.Dir(fileName)) err := os.MkdirAll(poolPath, 0777) if err != nil { diff --git a/files/public_test.go b/files/public_test.go index 16f724cb..f135bd3e 100644 --- a/files/public_test.go +++ b/files/public_test.go @@ -233,7 +233,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Assert(err, IsNil) // Test using hardlinks - err = s.storage.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false) + err = s.storage.LinkFromPool(t.prefix, t.publishedDirectory, t.sourcePath, pool, srcPoolPath, sourceChecksum, false) c.Assert(err, IsNil) st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename)) @@ -243,7 +243,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Check(int(info.Nlink), Equals, 3) // Test using symlinks - err = s.storageSymlink.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false) + err = s.storageSymlink.LinkFromPool(t.prefix, t.publishedDirectory, t.sourcePath, pool, srcPoolPath, sourceChecksum, false) c.Assert(err, IsNil) st, err = os.Lstat(filepath.Join(s.storageSymlink.rootPath, t.prefix, t.expectedFilename)) @@ -254,7 +254,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Check(int(info.Mode&syscall.S_IFMT), Equals, int(syscall.S_IFLNK)) // Test using copy with checksum verification - err = s.storageCopy.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false) + err = s.storageCopy.LinkFromPool(t.prefix, t.publishedDirectory, t.sourcePath, pool, srcPoolPath, sourceChecksum, false) c.Assert(err, IsNil) st, err = os.Stat(filepath.Join(s.storageCopy.rootPath, t.prefix, t.expectedFilename)) @@ -264,7 +264,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Check(int(info.Nlink), Equals, 1) // Test using copy with size verification - err = s.storageCopySize.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false) + err = s.storageCopySize.LinkFromPool(t.prefix, t.publishedDirectory, t.sourcePath, pool, srcPoolPath, sourceChecksum, false) c.Assert(err, IsNil) st, err = os.Stat(filepath.Join(s.storageCopySize.rootPath, t.prefix, t.expectedFilename)) @@ -289,7 +289,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Assert(err, IsNil) nlinks := int(st.Sys().(*syscall.Stat_t).Nlink) - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false) + err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false) c.Check(err, ErrorMatches, ".*file already exists and is different") st, err = pool.Stat(srcPoolPath) @@ -297,7 +297,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Check(int(st.Sys().(*syscall.Stat_t).Nlink), Equals, nlinks) // linking with force - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true) + err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true) c.Check(err, IsNil) st, err = pool.Stat(srcPoolPath) @@ -305,21 +305,21 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Check(int(st.Sys().(*syscall.Stat_t).Nlink), Equals, nlinks+1) // Test using symlinks - err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false) + err = s.storageSymlink.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false) c.Check(err, ErrorMatches, ".*file already exists and is different") - err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true) + err = s.storageSymlink.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true) c.Check(err, IsNil) // Test using copy with checksum verification - err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false) + err = s.storageCopy.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false) c.Check(err, ErrorMatches, ".*file already exists and is different") - err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true) + err = s.storageCopy.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true) c.Check(err, IsNil) // Test using copy with size verification (this will NOT detect the difference) - err = s.storageCopySize.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false) + err = s.storageCopySize.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false) c.Check(err, IsNil) } diff --git a/s3/public.go b/s3/public.go index 8c162cd8..4dc7c959 100644 --- a/s3/public.go +++ b/s3/public.go @@ -305,20 +305,21 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error // LinkFromPool links package file from pool to dist's pool location // -// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/) +// publishedPrefix is desired prefix for the location in the pool. +// publishedRelParh is desired location in pool (like pool/component/liba/libav/) // sourcePool is instance of aptly.PackagePool // sourcePath is filepath to package file in package pool // // LinkFromPool returns relative path for the published file to be included in package index -func (storage *PublishedStorage) LinkFromPool(prefix string, path string, fileName string, sourcePool aptly.PackagePool, +func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { - publishedDirectory := filepath.Join(prefix, path) + publishedDirectory := filepath.Join(publishedPrefix, publishedRelPath) relPath := filepath.Join(publishedDirectory, fileName) poolPath := filepath.Join(storage.prefix, relPath) if storage.pathCache == nil { - paths, md5s, err := storage.internalFilelist(filepath.Join(storage.prefix, prefix, "pool"), true) + paths, md5s, err := storage.internalFilelist(filepath.Join(storage.prefix, publishedPrefix, "pool"), true) if err != nil { return errors.Wrap(err, "error caching paths under prefix") } diff --git a/s3/public_test.go b/s3/public_test.go index 2e676234..a3586325 100644 --- a/s3/public_test.go +++ b/s3/public_test.go @@ -270,50 +270,124 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Assert(err, IsNil) // first link from pool - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) c.Check(err, IsNil) c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) // duplicate link from pool - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) c.Check(err, IsNil) c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) // link from pool with conflict - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false) + err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false) c.Check(err, ErrorMatches, ".*file already exists and is different.*") c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) // link from pool with conflict and force - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true) + err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true) c.Check(err, IsNil) c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam")) // for prefixed storage: // first link from pool - err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) c.Check(err, IsNil) // 2nd link from pool, providing wrong path for source file // // this test should check that file already exists in S3 and skip upload (which would fail if not skipped) s.prefixedStorage.pathCache = nil - err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false) + err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false) c.Check(err, IsNil) c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents")) // link from pool with nested file name - err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false) + err = s.storage.LinkFromPool("", "dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false) c.Check(err, IsNil) c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents")) } +func (s *PublishedStorageSuite) TestLinkFromPoolCache(c *C) { + root := c.MkDir() + pool := files.NewPackagePool(root, false) + cs := files.NewMockChecksumStorage() + + tmpFile1 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb") + err := ioutil.WriteFile(tmpFile1, []byte("Contents"), 0644) + c.Assert(err, IsNil) + cksum1 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"} + + src1, err := pool.Import(tmpFile1, "mars-invaders_1.03.deb", &cksum1, true, cs) + c.Assert(err, IsNil) + + // Publish two packages at the same publish prefix + err = s.storage.LinkFromPool("", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + c.Check(err, IsNil) + + err = s.storage.LinkFromPool("", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + c.Check(err, IsNil) + + // Check only one listing request was done to the server + s.checkGetRequestsEqual(c, "/test?", []string{"/test?max-keys=1000&prefix="}) + + s.srv.Requests = nil + // Publish two packages at a different prefix + err = s.storage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + c.Check(err, IsNil) + + err = s.storage.LinkFromPool("publish-prefix", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + c.Check(err, IsNil) + + // Check only one listing request was done to the server + s.checkGetRequestsEqual(c, "/test?", []string{ + "/test?max-keys=1000&prefix=publish-prefix%2F", + }) + + s.srv.Requests = nil + // Publish two packages at a prefixed storage + err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + c.Check(err, IsNil) + + err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + c.Check(err, IsNil) + + // Check only one listing request was done to the server + s.checkGetRequestsEqual(c, "/test?", []string{ + "/test?max-keys=1000&prefix=lala%2F", + }) + + s.srv.Requests = nil + // Publish two packages at a prefixed storage plus a publish prefix. + err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + c.Check(err, IsNil) + + err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "b"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + c.Check(err, IsNil) + + // Check only one listing request was done to the server + s.checkGetRequestsEqual(c, "/test?", []string{ + "/test?max-keys=1000&prefix=lala%2Fpublish-prefix%2F", + }) + + // This step checks that files already exists in S3 and skip upload (which would fail if not skipped). + s.prefixedStorage.pathCache = nil + err = s.prefixedStorage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false) + c.Check(err, IsNil) + err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false) + c.Check(err, IsNil) + err = s.storage.LinkFromPool("publish-prefix", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false) + c.Check(err, IsNil) + err = s.storage.LinkFromPool("", filepath.Join("pool", "a"), "mars-invaders_1.03.deb", pool, "non-existent-file", cksum1, false) + c.Check(err, IsNil) +} + func (s *PublishedStorageSuite) TestSymLink(c *C) { s.PutFile(c, "a/b", []byte("test")) diff --git a/swift/public.go b/swift/public.go index 057b88a5..7732712d 100644 --- a/swift/public.go +++ b/swift/public.go @@ -188,16 +188,16 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error // LinkFromPool links package file from pool to dist's pool location // -// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/) +// publishedPrefix is desired prefix for the location in the pool. +// publishedRelParh is desired location in pool (like pool/component/liba/libav/) // sourcePool is instance of aptly.PackagePool // sourcePath is filepath to package file in package pool // // LinkFromPool returns relative path for the published file to be included in package index -func (storage *PublishedStorage) LinkFromPool(prefix string, path string, fileName string, sourcePool aptly.PackagePool, +func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { - publishedDirectory := filepath.Join(prefix, path) - relPath := filepath.Join(publishedDirectory, fileName) + relPath := filepath.Join(publishedPrefix, publishedRelPath, fileName) poolPath := filepath.Join(storage.prefix, relPath) var ( diff --git a/swift/public_test.go b/swift/public_test.go index ed8ff608..620b94a2 100644 --- a/swift/public_test.go +++ b/swift/public_test.go @@ -169,7 +169,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Assert(err, IsNil) // first link from pool - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) c.Check(err, IsNil) data, err := s.storage.conn.ObjectGetBytes("test", "pool/main/m/mars-invaders/mars-invaders_1.03.deb") @@ -177,7 +177,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Check(data, DeepEquals, []byte("Contents")) // duplicate link from pool - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) + err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false) c.Check(err, IsNil) data, err = s.storage.conn.ObjectGetBytes("test", "pool/main/m/mars-invaders/mars-invaders_1.03.deb") @@ -185,7 +185,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Check(data, DeepEquals, []byte("Contents")) // link from pool with conflict - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false) + err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false) c.Check(err, ErrorMatches, ".*file already exists and is different.*") data, err = s.storage.conn.ObjectGetBytes("test", "pool/main/m/mars-invaders/mars-invaders_1.03.deb") @@ -193,7 +193,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Check(data, DeepEquals, []byte("Contents")) // link from pool with conflict and force - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true) + err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true) c.Check(err, IsNil) data, err = s.storage.conn.ObjectGetBytes("test", "pool/main/m/mars-invaders/mars-invaders_1.03.deb") @@ -201,7 +201,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Check(data, DeepEquals, []byte("Spam")) // link from pool with nested file name - err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false) + err = s.storage.LinkFromPool("", "dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false) c.Check(err, IsNil) data, err = s.storage.conn.ObjectGetBytes("test", "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz")