vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-06-09 06:04:12 +00:00
Code Issues Projects Releases Wiki Activity

Cache bucket content by prefix

Browse Source 
When a publishing uses a publish prefix, instead of listing the contents
of the whole bucket under the storage prefix, only list the contents of
the bucket under the storage prefix and publish prefix, and cache it by
publish prefix.
This speeds up publish operations under a prefix.
This commit is contained in:
Ludovico Cavedon
2023-04-11 10:26:29 -07:00
committed by André Roth
parent 01893a492f
commit fad660450c
9 changed files with 136 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
files/public.go
+4 -4
View File
@@ -118,17 +118,17 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
// LinkFromPool links package file from pool to dist's pool location
//
// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
// publishedPrefix is desired prefix for the location in the pool.
// publishedRelParh is desired location in pool (like pool/component/liba/libav/)
// sourcePool is instance of aptly.PackagePool
// sourcePath is a relative path to package file in package pool
//
// LinkFromPool returns relative path for the published file to be included in package index
func (storage *PublishedStorage) LinkFromPool(prefix string, path string, fileName string, sourcePool aptly.PackagePool,
func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool,
sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
publishedDirectory := filepath.Join(prefix, path)
baseName := filepath.Base(fileName)
poolPath := filepath.Join(storage.rootPath, publishedDirectory, filepath.Dir(fileName))
poolPath := filepath.Join(storage.rootPath, publishedPrefix, publishedRelPath, filepath.Dir(fileName))
err := os.MkdirAll(poolPath, 0777)
if err != nil {
files/public_test.go
+11 -11
View File
@@ -233,7 +233,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
c.Assert(err, IsNil)
// Test using hardlinks
err = s.storage.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
err = s.storage.LinkFromPool(t.prefix, t.publishedDirectory, t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
c.Assert(err, IsNil)
st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename))
@@ -243,7 +243,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
c.Check(int(info.Nlink), Equals, 3)
// Test using symlinks
err = s.storageSymlink.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
err = s.storageSymlink.LinkFromPool(t.prefix, t.publishedDirectory, t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
c.Assert(err, IsNil)
st, err = os.Lstat(filepath.Join(s.storageSymlink.rootPath, t.prefix, t.expectedFilename))
@@ -254,7 +254,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
c.Check(int(info.Mode&syscall.S_IFMT), Equals, int(syscall.S_IFLNK))
// Test using copy with checksum verification
err = s.storageCopy.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
err = s.storageCopy.LinkFromPool(t.prefix, t.publishedDirectory, t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
c.Assert(err, IsNil)
st, err = os.Stat(filepath.Join(s.storageCopy.rootPath, t.prefix, t.expectedFilename))
@@ -264,7 +264,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
c.Check(int(info.Nlink), Equals, 1)
// Test using copy with size verification
err = s.storageCopySize.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
err = s.storageCopySize.LinkFromPool(t.prefix, t.publishedDirectory, t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
c.Assert(err, IsNil)
st, err = os.Stat(filepath.Join(s.storageCopySize.rootPath, t.prefix, t.expectedFilename))
@@ -289,7 +289,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
c.Assert(err, IsNil)
nlinks := int(st.Sys().(*syscall.Stat_t).Nlink)
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
c.Check(err, ErrorMatches, ".*file already exists and is different")
st, err = pool.Stat(srcPoolPath)
@@ -297,7 +297,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
c.Check(int(st.Sys().(*syscall.Stat_t).Nlink), Equals, nlinks)
// linking with force
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
c.Check(err, IsNil)
st, err = pool.Stat(srcPoolPath)
@@ -305,21 +305,21 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
c.Check(int(st.Sys().(*syscall.Stat_t).Nlink), Equals, nlinks+1)
// Test using symlinks
err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
err = s.storageSymlink.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
c.Check(err, ErrorMatches, ".*file already exists and is different")
err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
err = s.storageSymlink.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
c.Check(err, IsNil)
// Test using copy with checksum verification
err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
err = s.storageCopy.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
c.Check(err, ErrorMatches, ".*file already exists and is different")
err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
err = s.storageCopy.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
c.Check(err, IsNil)
// Test using copy with size verification (this will NOT detect the difference)
err = s.storageCopySize.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
err = s.storageCopySize.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
c.Check(err, IsNil)
}