From 7bad358408f39ba73a88cda47fe3c57588255f4c Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 6 Apr 2017 20:06:15 +0300 Subject: [PATCH] Local package pool and local publishing rewritten with new constraints Local package pool now implements more generic package pool API. The base idea is to never expose full paths to files, so that other kinds of package pools (e.g. package pool in S3) could be used to implement the same interface. Files get into the pool only using `Import` method. `Import` method is now more smart, it supports moving files into the pool, it can detect if files reside on the same filesystem and use hardlinking instead of copying. This will make direct mirror downloads still as fast as they were with previous version which was performing download directly to package pool. New package pool doesn't have two things implemented yet: 1. New file placement according to SHA256 or other configured hash 2. Calculate at least SHA256/MD5 for each imported files. MD5 would be required for S3/Swift publishing --- files/package_pool.go | 162 ++++++++++++++++++++++++++++--------- files/package_pool_test.go | 124 ++++++++++++++++++++++------ files/public.go | 40 +++++++-- files/public_test.go | 74 ++++++++--------- 4 files changed, 294 insertions(+), 106 deletions(-) diff --git a/files/package_pool.go b/files/package_pool.go index da41f6e4..b3b27548 100644 --- a/files/package_pool.go +++ b/files/package_pool.go @@ -7,6 +7,9 @@ import ( "os" "path/filepath" "sync" + "syscall" + + "github.com/smira/go-uuid/uuid" "github.com/smira/aptly/aptly" "github.com/smira/aptly/utils" @@ -15,21 +18,33 @@ import ( // PackagePool is deduplicated storage of package files on filesystem type PackagePool struct { sync.Mutex - rootPath string + + rootPath string + supportLegacyPaths bool } // Check interface var ( - _ aptly.PackagePool = (*PackagePool)(nil) + _ aptly.PackagePool = (*PackagePool)(nil) + _ aptly.LocalPackagePool = (*PackagePool)(nil) ) // NewPackagePool creates new instance of PackagePool which specified root func NewPackagePool(root string) *PackagePool { - return &PackagePool{rootPath: filepath.Join(root, "pool")} + rootPath := filepath.Join(root, "pool") + rootPath, err := filepath.Abs(rootPath) + if err != nil { + panic(err) + } + + return &PackagePool{ + rootPath: rootPath, + supportLegacyPaths: true, + } } -// RelativePath returns path relative to pool's root for package files given checksum info and original filename -func (pool *PackagePool) RelativePath(filename string, checksums utils.ChecksumInfo) (string, error) { +// LegacyPath returns path relative to pool's root for pre-1.1 aptly (based on MD5) +func (pool *PackagePool) LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error) { filename = filepath.Base(filename) if filename == "." || filename == "/" { return "", fmt.Errorf("filename %s is invalid", filename) @@ -44,16 +59,6 @@ func (pool *PackagePool) RelativePath(filename string, checksums utils.ChecksumI return filepath.Join(hashMD5[0:2], hashMD5[2:4], filename), nil } -// Path returns full path to package file in pool given filename and hash of file contents -func (pool *PackagePool) Path(filename string, checksums utils.ChecksumInfo) (string, error) { - relative, err := pool.RelativePath(filename, checksums) - if err != nil { - return "", err - } - - return filepath.Join(pool.rootPath, relative), nil -} - // FilepathList returns file paths of all the files in the pool func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) { pool.Lock() @@ -117,56 +122,141 @@ func (pool *PackagePool) Remove(path string) (size int64, err error) { } // Import copies file into package pool -func (pool *PackagePool) Import(path string, checksums utils.ChecksumInfo) error { +// +// - srcPath is full path to source file as it is now +// - basename is desired human-readable name (canonical filename) +// - checksums are used to calculate file placement +// - move indicates whether srcPath can be removed +func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool) (string, error) { pool.Lock() defer pool.Unlock() - source, err := os.Open(path) + source, err := os.Open(srcPath) if err != nil { - return err + return "", err } defer source.Close() sourceInfo, err := source.Stat() if err != nil { - return err + return "", err } - poolPath, err := pool.Path(path, checksums) + // build target path + // TODO: replace with new build scheme + poolPath, err := pool.LegacyPath(basename, checksums) if err != nil { - return err + return "", err } - targetInfo, err := os.Stat(poolPath) + fullPoolPath := filepath.Join(pool.rootPath, poolPath) + + targetInfo, err := os.Stat(fullPoolPath) if err != nil { if !os.IsNotExist(err) { // unable to stat target location? - return err + return "", err } } else { - // target already exists - if targetInfo.Size() != sourceInfo.Size() { - // trying to overwrite file? - return fmt.Errorf("unable to import into pool: file %s already exists", poolPath) + // target already exists and same size + if targetInfo.Size() == sourceInfo.Size() { + return poolPath, nil } - // assume that target is already there - return nil + // trying to overwrite file? + return "", fmt.Errorf("unable to import into pool: file %s already exists", poolPath) + } + + if pool.supportLegacyPaths { + // file doesn't exist at new location, check legacy location + var ( + legacyTargetInfo os.FileInfo + legacyPath, legacyFullPath string + ) + + legacyPath, err = pool.LegacyPath(basename, checksums) + if err != nil { + return "", err + } + legacyFullPath = filepath.Join(pool.rootPath, legacyPath) + + legacyTargetInfo, err = os.Stat(legacyFullPath) + if err != nil { + if !os.IsNotExist(err) { + return "", err + } + } else { + // legacy file exists + if legacyTargetInfo.Size() == sourceInfo.Size() { + // file exists at legacy path and it's same size, consider it's already in the pool + return legacyPath, nil + } + + // size is different, import at new path + } } // create subdirs as necessary - err = os.MkdirAll(filepath.Dir(poolPath), 0777) + poolDir := filepath.Dir(fullPoolPath) + err = os.MkdirAll(poolDir, 0777) if err != nil { - return err + return "", err } - target, err := os.Create(poolPath) + // check if we can use hardlinks instead of copying/moving + poolDirInfo, err := os.Stat(poolDir) if err != nil { - return err + return "", err } - defer target.Close() - _, err = io.Copy(target, source) + if poolDirInfo.Sys().(*syscall.Stat_t).Dev == sourceInfo.Sys().(*syscall.Stat_t).Dev { + // same filesystem, try to use hardlink + err = os.Link(srcPath, fullPoolPath) + } else { + err = os.ErrInvalid + } - return err + if err != nil { + // different filesystems or failed hardlink, fallback to copy + var target *os.File + target, err = os.Create(fullPoolPath) + if err != nil { + return "", err + } + defer target.Close() + + _, err = io.Copy(target, source) + + if err == nil { + err = target.Close() + } + } + + if err == nil && move { + err = os.Remove(srcPath) + } + + return poolPath, err +} + +// Open returns io.ReadCloser to access the file +func (pool *PackagePool) Open(path string) (io.ReadCloser, error) { + return os.Open(filepath.Join(pool.rootPath, path)) +} + +// Stat returns Unix stat(2) info +func (pool *PackagePool) Stat(path string) (os.FileInfo, error) { + return os.Stat(filepath.Join(pool.rootPath, path)) +} + +// Link generates hardlink to destination path +func (pool *PackagePool) Link(path, dstPath string) error { + return os.Link(filepath.Join(pool.rootPath, path), dstPath) +} + +// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on) +func (pool *PackagePool) GenerateTempPath(filename string) (string, error) { + random := uuid.NewRandom().String() + + return filepath.Join(pool.rootPath, random[0:2], random[2:4], random[4:]+filename), nil } diff --git a/files/package_pool_test.go b/files/package_pool_test.go index 316aa0cd..3b9e583f 100644 --- a/files/package_pool_test.go +++ b/files/package_pool_test.go @@ -1,10 +1,12 @@ package files import ( + "io" "io/ioutil" "os" "path/filepath" "runtime" + "syscall" "github.com/smira/aptly/utils" @@ -14,6 +16,7 @@ import ( type PackagePoolSuite struct { pool *PackagePool checksum utils.ChecksumInfo + debFile string } var _ = Suite(&PackagePoolSuite{}) @@ -23,30 +26,23 @@ func (s *PackagePoolSuite) SetUpTest(c *C) { s.checksum = utils.ChecksumInfo{ MD5: "91b1a1480b90b9e269ca44d897b12575", } + _, _File, _, _ := runtime.Caller(0) + s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb") } -func (s *PackagePoolSuite) TestRelativePath(c *C) { - path, err := s.pool.RelativePath("a/b/package.deb", s.checksum) +func (s *PackagePoolSuite) TestLegacyPath(c *C) { + path, err := s.pool.LegacyPath("a/b/package.deb", &s.checksum) c.Assert(err, IsNil) c.Assert(path, Equals, "91/b1/package.deb") - _, err = s.pool.RelativePath("/", s.checksum) + _, err = s.pool.LegacyPath("/", &s.checksum) c.Assert(err, ErrorMatches, ".*is invalid") - _, err = s.pool.RelativePath("", s.checksum) + _, err = s.pool.LegacyPath("", &s.checksum) c.Assert(err, ErrorMatches, ".*is invalid") - _, err = s.pool.RelativePath("a/b/package.deb", utils.ChecksumInfo{MD5: "9"}) + _, err = s.pool.LegacyPath("a/b/package.deb", &utils.ChecksumInfo{MD5: "9"}) c.Assert(err, ErrorMatches, ".*MD5 is missing") } -func (s *PackagePoolSuite) TestPath(c *C) { - path, err := s.pool.Path("a/b/package.deb", s.checksum) - c.Assert(err, IsNil) - c.Assert(path, Equals, filepath.Join(s.pool.rootPath, "91/b1/package.deb")) - - _, err = s.pool.Path("/", s.checksum) - c.Assert(err, ErrorMatches, ".*is invalid") -} - func (s *PackagePoolSuite) TestFilepathList(c *C) { list, err := s.pool.FilepathList(nil) c.Check(err, IsNil) @@ -93,33 +89,109 @@ func (s *PackagePoolSuite) TestRemove(c *C) { } func (s *PackagePoolSuite) TestImportOk(c *C) { - _, _File, _, _ := runtime.Caller(0) - debFile := filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb") - - err := s.pool.Import(debFile, s.checksum) + path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false) c.Check(err, IsNil) + c.Check(path, Equals, "91/b1/libboost-program-options-dev_1.49.0.1_i386.deb") - info, err := os.Stat(filepath.Join(s.pool.rootPath, "91", "b1", "libboost-program-options-dev_1.49.0.1_i386.deb")) - c.Check(err, IsNil) + info, err := s.pool.Stat(path) + c.Assert(err, IsNil) c.Check(info.Size(), Equals, int64(2738)) + c.Check(info.Sys().(*syscall.Stat_t).Nlink > 1, Equals, true) + + // import as different name + path, err = s.pool.Import(s.debFile, "some.deb", &s.checksum, false) + c.Check(err, IsNil) + c.Check(path, Equals, "91/b1/some.deb") // double import, should be ok - err = s.pool.Import(debFile, s.checksum) + path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false) c.Check(err, IsNil) + c.Check(path, Equals, "91/b1/libboost-program-options-dev_1.49.0.1_i386.deb") +} + +func (s *PackagePoolSuite) TestImportMove(c *C) { + tmpDir := c.MkDir() + tmpPath := filepath.Join(tmpDir, filepath.Base(s.debFile)) + + dst, err := os.Create(tmpPath) + c.Assert(err, IsNil) + + src, err := os.Open(s.debFile) + c.Assert(err, IsNil) + + _, err = io.Copy(dst, src) + c.Assert(err, IsNil) + + c.Assert(dst.Close(), IsNil) + c.Assert(src.Close(), IsNil) + + path, err := s.pool.Import(tmpPath, filepath.Base(tmpPath), &s.checksum, true) + c.Check(err, IsNil) + c.Check(path, Equals, "91/b1/libboost-program-options-dev_1.49.0.1_i386.deb") + + info, err := s.pool.Stat(path) + c.Assert(err, IsNil) + c.Check(info.Size(), Equals, int64(2738)) + c.Check(info.Sys().(*syscall.Stat_t).Nlink, Equals, uint16(1)) } func (s *PackagePoolSuite) TestImportNotExist(c *C) { - err := s.pool.Import("no-such-file", s.checksum) + _, err := s.pool.Import("no-such-file", "a.deb", &s.checksum, false) c.Check(err, ErrorMatches, ".*no such file or directory") } func (s *PackagePoolSuite) TestImportOverwrite(c *C) { - _, _File, _, _ := runtime.Caller(0) - debFile := filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb") - os.MkdirAll(filepath.Join(s.pool.rootPath, "91", "b1"), 0755) ioutil.WriteFile(filepath.Join(s.pool.rootPath, "91", "b1", "libboost-program-options-dev_1.49.0.1_i386.deb"), []byte("1"), 0644) - err := s.pool.Import(debFile, s.checksum) + _, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false) c.Check(err, ErrorMatches, "unable to import into pool.*") } + +func (s *PackagePoolSuite) TestStat(c *C) { + path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false) + c.Check(err, IsNil) + + info, err := s.pool.Stat(path) + c.Assert(err, IsNil) + c.Check(info.Size(), Equals, int64(2738)) + + _, err = s.pool.Stat("do/es/ntexist") + c.Assert(os.IsNotExist(err), Equals, true) +} + +func (s *PackagePoolSuite) TestOpen(c *C) { + path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false) + c.Check(err, IsNil) + + f, err := s.pool.Open(path) + c.Assert(err, IsNil) + contents, err := ioutil.ReadAll(f) + c.Assert(err, IsNil) + c.Check(len(contents), Equals, 2738) + c.Check(f.Close(), IsNil) + + _, err = s.pool.Open("do/es/ntexist") + c.Assert(os.IsNotExist(err), Equals, true) +} + +func (s *PackagePoolSuite) TestLink(c *C) { + path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false) + c.Check(err, IsNil) + + tmpDir := c.MkDir() + dstPath := filepath.Join(tmpDir, filepath.Base(s.debFile)) + c.Check(s.pool.Link(path, dstPath), IsNil) + + info, err := os.Stat(dstPath) + c.Assert(err, IsNil) + c.Check(info.Size(), Equals, int64(2738)) + c.Check(info.Sys().(*syscall.Stat_t).Nlink > 2, Equals, true) +} + +func (s *PackagePoolSuite) TestGenerateRandomPath(c *C) { + path, err := s.pool.GenerateTempPath("a.deb") + c.Check(err, IsNil) + + c.Check(path, Matches, ".+/[0-9a-f][0-9a-f]/[0-9a-f][0-9a-f]/[0-9a-f-]+a\\.deb") +} diff --git a/files/public.go b/files/public.go index 68c83759..73cac95e 100644 --- a/files/public.go +++ b/files/public.go @@ -114,13 +114,11 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress // // publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/) // sourcePool is instance of aptly.PackagePool -// sourcePath is filepath to package file in package pool +// sourcePath is a relative path to package file in package pool // // LinkFromPool returns relative path for the published file to be included in package index func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { - // verify that package pool is local pool is filesystem pool - _ = sourcePool.(*PackagePool) baseName := filepath.Base(sourcePath) poolPath := filepath.Join(storage.rootPath, publishedDirectory) @@ -135,7 +133,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP dstStat, err = os.Stat(filepath.Join(poolPath, baseName)) if err == nil { // already exists, check source file - srcStat, err = os.Stat(sourcePath) + srcStat, err = sourcePool.Stat(sourcePath) if err != nil { // source file doesn't exist? problem! return err @@ -184,13 +182,39 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP } } - // destination doesn't exist (or forced), create link + // destination doesn't exist (or forced), create link or copy if storage.linkMethod == LinkMethodCopy { - err = utils.CopyFile(sourcePath, filepath.Join(poolPath, baseName)) + var r aptly.ReadSeekerCloser + r, err = sourcePool.Open(sourcePath) + if err != nil { + return err + } + + var dst *os.File + dst, err = os.Create(filepath.Join(poolPath, baseName)) + if err != nil { + r.Close() + return err + } + + _, err = io.Copy(dst, r) + if err != nil { + r.Close() + dst.Close() + return err + } + + err = r.Close() + if err != nil { + dst.Close() + return err + } + + err = dst.Close() } else if storage.linkMethod == LinkMethodSymLink { - err = os.Symlink(sourcePath, filepath.Join(poolPath, baseName)) + err = sourcePool.(aptly.LocalPackagePool).Symlink(sourcePath, filepath.Join(poolPath, baseName)) } else { - err = os.Link(sourcePath, filepath.Join(poolPath, baseName)) + err = sourcePool.(aptly.LocalPackagePool).Link(sourcePath, filepath.Join(poolPath, baseName)) } return err diff --git a/files/public_test.go b/files/public_test.go index 27717b6a..80c1f4ca 100644 --- a/files/public_test.go +++ b/files/public_test.go @@ -108,6 +108,7 @@ func (s *PublishedStorageSuite) TestRemoveDirs(c *C) { c.Assert(err, IsNil) err = s.storage.RemoveDirs("ppa/dists/", nil) + c.Assert(err, IsNil) _, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release")) c.Assert(err, NotNil) @@ -122,6 +123,7 @@ func (s *PublishedStorageSuite) TestRemove(c *C) { c.Assert(err, IsNil) err = s.storage.Remove("ppa/dists/squeeze/Release") + c.Assert(err, IsNil) _, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release")) c.Assert(err, NotNil) @@ -139,28 +141,28 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { { // package name regular prefix: "", component: "main", - sourcePath: "pool/01/ae/mars-invaders_1.03.deb", + sourcePath: "mars-invaders_1.03.deb", poolDirectory: "m/mars-invaders", expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb", }, { // lib-like filename prefix: "", component: "main", - sourcePath: "pool/01/ae/libmars-invaders_1.03.deb", + sourcePath: "libmars-invaders_1.03.deb", poolDirectory: "libm/libmars-invaders", expectedFilename: "pool/main/libm/libmars-invaders/libmars-invaders_1.03.deb", }, { // duplicate link, shouldn't panic prefix: "", component: "main", - sourcePath: "pool/01/ae/mars-invaders_1.03.deb", + sourcePath: "mars-invaders_1.03.deb", poolDirectory: "m/mars-invaders", expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb", }, { // prefix & component prefix: "ppa", component: "contrib", - sourcePath: "pool/01/ae/libmars-invaders_1.04.deb", + sourcePath: "libmars-invaders_1.04.deb", poolDirectory: "libm/libmars-invaders", expectedFilename: "pool/contrib/libm/libmars-invaders/libmars-invaders_1.04.deb", }, @@ -169,38 +171,39 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { pool := NewPackagePool(s.root) for _, t := range tests { - t.sourcePath = filepath.Join(s.root, t.sourcePath) - - err := os.MkdirAll(filepath.Dir(t.sourcePath), 0755) + tmpPath := filepath.Join(c.MkDir(), t.sourcePath) + err := ioutil.WriteFile(tmpPath, []byte("Contents"), 0644) c.Assert(err, IsNil) - err = ioutil.WriteFile(t.sourcePath, []byte("Contents"), 0644) + sourceChecksum, err := utils.ChecksumsForFile(tmpPath) c.Assert(err, IsNil) - sourceChecksum, err := utils.ChecksumsForFile(t.sourcePath) + srcPoolPath, err := pool.Import(tmpPath, t.sourcePath, &utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}, false) c.Assert(err, IsNil) - err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, sourceChecksum, false) + // Test using hardlinks + err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, srcPoolPath, sourceChecksum, false) c.Assert(err, IsNil) st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename)) c.Assert(err, IsNil) info := st.Sys().(*syscall.Stat_t) - c.Check(int(info.Nlink), Equals, 2) + c.Check(int(info.Nlink), Equals, 3) // Test using symlinks - err = s.storageSymlink.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, sourceChecksum, false) + err = s.storageSymlink.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, srcPoolPath, sourceChecksum, false) c.Assert(err, IsNil) - st, err = os.Stat(filepath.Join(s.storageSymlink.rootPath, t.prefix, t.expectedFilename)) + st, err = os.Lstat(filepath.Join(s.storageSymlink.rootPath, t.prefix, t.expectedFilename)) c.Assert(err, IsNil) info = st.Sys().(*syscall.Stat_t) - c.Check(int(info.Nlink), Equals, 2) + c.Check(int(info.Nlink), Equals, 1) + c.Check(int(info.Mode&syscall.S_IFMT), Equals, int(syscall.S_IFLNK)) // Test using copy with checksum verification - err = s.storageCopy.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, sourceChecksum, false) + err = s.storageCopy.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, srcPoolPath, sourceChecksum, false) c.Assert(err, IsNil) st, err = os.Stat(filepath.Join(s.storageCopy.rootPath, t.prefix, t.expectedFilename)) @@ -210,7 +213,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { c.Check(int(info.Nlink), Equals, 1) // Test using copy with size verification - err = s.storageCopySize.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, sourceChecksum, false) + err = s.storageCopySize.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, srcPoolPath, sourceChecksum, false) c.Assert(err, IsNil) st, err = os.Stat(filepath.Join(s.storageCopySize.rootPath, t.prefix, t.expectedFilename)) @@ -221,51 +224,50 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { } // test linking files to duplicate final name - sourcePath := filepath.Join(s.root, "pool/02/bc/mars-invaders_1.03.deb") - err := os.MkdirAll(filepath.Dir(sourcePath), 0755) + tmpPath := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb") + err := ioutil.WriteFile(tmpPath, []byte("cONTENTS"), 0644) c.Assert(err, IsNil) - // use same size to ensure copy with size check will fail on this one - err = ioutil.WriteFile(sourcePath, []byte("cONTENTS"), 0644) + sourceChecksum, err := utils.ChecksumsForFile(tmpPath) c.Assert(err, IsNil) - sourceChecksum, err := utils.ChecksumsForFile(sourcePath) + srcPoolPath, err := pool.Import(tmpPath, "mars-invaders_1.03.deb", &utils.ChecksumInfo{MD5: "02bcda7a1ce305a3b60af9d5733ac1d"}, true) c.Assert(err, IsNil) - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, false) + st, err := pool.Stat(srcPoolPath) + c.Assert(err, IsNil) + nlinks := int(st.Sys().(*syscall.Stat_t).Nlink) + + err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, srcPoolPath, sourceChecksum, false) c.Check(err, ErrorMatches, ".*file already exists and is different") - st, err := os.Stat(sourcePath) + st, err = pool.Stat(srcPoolPath) c.Assert(err, IsNil) - - info := st.Sys().(*syscall.Stat_t) - c.Check(int(info.Nlink), Equals, 1) + c.Check(int(st.Sys().(*syscall.Stat_t).Nlink), Equals, nlinks) // linking with force - err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, true) + err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, srcPoolPath, sourceChecksum, true) c.Check(err, IsNil) - st, err = os.Stat(sourcePath) + st, err = pool.Stat(srcPoolPath) c.Assert(err, IsNil) - - info = st.Sys().(*syscall.Stat_t) - c.Check(int(info.Nlink), Equals, 2) + c.Check(int(st.Sys().(*syscall.Stat_t).Nlink), Equals, nlinks+1) // Test using symlinks - err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, false) + err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, srcPoolPath, sourceChecksum, false) c.Check(err, ErrorMatches, ".*file already exists and is different") - err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, true) + err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, srcPoolPath, sourceChecksum, true) c.Check(err, IsNil) // Test using copy with checksum verification - err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, false) + err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, srcPoolPath, sourceChecksum, false) c.Check(err, ErrorMatches, ".*file already exists and is different") - err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, true) + err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, srcPoolPath, sourceChecksum, true) c.Check(err, IsNil) // Test using copy with size verification (this will NOT detect the difference) - err = s.storageCopySize.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, false) + err = s.storageCopySize.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, srcPoolPath, sourceChecksum, false) c.Check(err, IsNil) }