mirror of
https://github.com/aptly-dev/aptly.git
synced 2026-06-06 05:30:57 +00:00
Implemented filesystem endpoint with support for hardlinks, symlinks and copy.
This commit is contained in:
Clemens Rabe
+78
-11
@@ -5,6 +5,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/smira/aptly/aptly"
|
||||
@@ -13,18 +14,53 @@ import (
|
||||
|
||||
// PublishedStorage abstract file system with public dirs (published repos)
|
||||
type PublishedStorage struct {
|
||||
rootPath string
|
||||
rootPath string
|
||||
linkMethod uint
|
||||
verifyMethod uint
|
||||
}
|
||||
|
||||
// Check interfaces
|
||||
var (
|
||||
_ aptly.PublishedStorage = (*PublishedStorage)(nil)
|
||||
_ aptly.LocalPublishedStorage = (*PublishedStorage)(nil)
|
||||
_ aptly.PublishedStorage = (*PublishedStorage)(nil)
|
||||
_ aptly.FileSystemPublishedStorage = (*PublishedStorage)(nil)
|
||||
)
|
||||
|
||||
// Constants defining the type of creating links
|
||||
const (
|
||||
LinkMethodHardLink uint = iota
|
||||
LinkMethodSymLink
|
||||
LinkMethodCopy
|
||||
)
|
||||
|
||||
// Constants defining the type of file verification for LinkMethodCopy
|
||||
const (
|
||||
VerificationMethodChecksum uint = iota
|
||||
VerificationMethodFileSize
|
||||
)
|
||||
|
||||
// NewPublishedStorage creates new instance of PublishedStorage which specified root
|
||||
func NewPublishedStorage(root string) *PublishedStorage {
|
||||
return &PublishedStorage{rootPath: filepath.Join(root, "public")}
|
||||
func NewPublishedStorage(root string, linkMethod string, verifyMethod string) *PublishedStorage {
|
||||
// Ensure linkMethod is one of 'hardlink', 'symlink', 'copy'
|
||||
var verifiedLinkMethod uint
|
||||
|
||||
if strings.EqualFold(linkMethod, "copy") {
|
||||
verifiedLinkMethod = LinkMethodCopy
|
||||
} else if strings.EqualFold(linkMethod, "symlink") {
|
||||
verifiedLinkMethod = LinkMethodSymLink
|
||||
} else {
|
||||
verifiedLinkMethod = LinkMethodHardLink
|
||||
}
|
||||
|
||||
var verifiedVerifyMethod uint
|
||||
|
||||
if strings.EqualFold(verifyMethod, "size") {
|
||||
verifiedVerifyMethod = VerificationMethodFileSize
|
||||
} else {
|
||||
verifiedVerifyMethod = VerificationMethodChecksum
|
||||
}
|
||||
|
||||
return &PublishedStorage{rootPath: root, linkMethod: verifiedLinkMethod,
|
||||
verifyMethod: verifiedVerifyMethod}
|
||||
}
|
||||
|
||||
// PublicPath returns root of public part
|
||||
@@ -105,12 +141,35 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
|
||||
return err
|
||||
}
|
||||
|
||||
srcSys := srcStat.Sys().(*syscall.Stat_t)
|
||||
dstSys := dstStat.Sys().(*syscall.Stat_t)
|
||||
if storage.linkMethod == LinkMethodCopy {
|
||||
if storage.verifyMethod == VerificationMethodFileSize {
|
||||
// if source and destination have the same size, no need to copy
|
||||
if srcStat.Size() == dstStat.Size() {
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
// if source and destination have the same checksums, no need to copy
|
||||
dstMD5, err := utils.MD5ChecksumForFile(filepath.Join(poolPath, baseName))
|
||||
|
||||
// source and destination inodes match, no need to link
|
||||
if srcSys.Ino == dstSys.Ino {
|
||||
return nil
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if dstMD5 == sourceChecksums.MD5 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
} else {
|
||||
srcSys := srcStat.Sys().(*syscall.Stat_t)
|
||||
dstSys := dstStat.Sys().(*syscall.Stat_t)
|
||||
|
||||
// if source and destination inodes match, no need to link
|
||||
|
||||
// Symlink can point to different filesystem with identical inodes
|
||||
// so we have to check the device as well.
|
||||
if srcSys.Ino == dstSys.Ino && srcSys.Dev == dstSys.Dev {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// source and destination have different inodes, if !forced, this is fatal error
|
||||
@@ -126,7 +185,15 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
|
||||
}
|
||||
|
||||
// destination doesn't exist (or forced), create link
|
||||
return os.Link(sourcePath, filepath.Join(poolPath, baseName))
|
||||
if storage.linkMethod == LinkMethodCopy {
|
||||
err = utils.CopyFile(sourcePath, filepath.Join(poolPath, baseName))
|
||||
} else if storage.linkMethod == LinkMethodSymLink {
|
||||
err = os.Symlink(sourcePath, filepath.Join(poolPath, baseName))
|
||||
} else {
|
||||
err = os.Link(sourcePath, filepath.Join(poolPath, baseName))
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Filelist returns list of files under prefix
|
||||
|
||||
+84
-8
@@ -12,19 +12,40 @@ import (
|
||||
)
|
||||
|
||||
type PublishedStorageSuite struct {
|
||||
root string
|
||||
storage *PublishedStorage
|
||||
root string
|
||||
storage *PublishedStorage
|
||||
storageSymlink *PublishedStorage
|
||||
storageCopy *PublishedStorage
|
||||
storageCopySize *PublishedStorage
|
||||
}
|
||||
|
||||
var _ = Suite(&PublishedStorageSuite{})
|
||||
|
||||
func (s *PublishedStorageSuite) SetUpTest(c *C) {
|
||||
s.root = c.MkDir()
|
||||
s.storage = NewPublishedStorage(s.root)
|
||||
s.storage = NewPublishedStorage(filepath.Join(s.root, "public"), "", "")
|
||||
s.storageSymlink = NewPublishedStorage(filepath.Join(s.root, "public_symlink"), "symlink", "")
|
||||
s.storageCopy = NewPublishedStorage(filepath.Join(s.root, "public_copy"), "copy", "")
|
||||
s.storageCopySize = NewPublishedStorage(filepath.Join(s.root, "public_copysize"), "copy", "size")
|
||||
}
|
||||
|
||||
func (s *PublishedStorageSuite) TestLinkMethodField(c *C) {
|
||||
c.Assert(s.storage.linkMethod, Equals, LinkMethodHardLink)
|
||||
c.Assert(s.storageSymlink.linkMethod, Equals, LinkMethodSymLink)
|
||||
c.Assert(s.storageCopy.linkMethod, Equals, LinkMethodCopy)
|
||||
c.Assert(s.storageCopySize.linkMethod, Equals, LinkMethodCopy)
|
||||
}
|
||||
|
||||
func (s *PublishedStorageSuite) TestVerifyMethodField(c *C) {
|
||||
c.Assert(s.storageCopy.verifyMethod, Equals, VerificationMethodChecksum)
|
||||
c.Assert(s.storageCopySize.verifyMethod, Equals, VerificationMethodFileSize)
|
||||
}
|
||||
|
||||
func (s *PublishedStorageSuite) TestPublicPath(c *C) {
|
||||
c.Assert(s.storage.PublicPath(), Equals, filepath.Join(s.root, "public"))
|
||||
c.Assert(s.storageSymlink.PublicPath(), Equals, filepath.Join(s.root, "public_symlink"))
|
||||
c.Assert(s.storageCopy.PublicPath(), Equals, filepath.Join(s.root, "public_copy"))
|
||||
c.Assert(s.storageCopySize.PublicPath(), Equals, filepath.Join(s.root, "public_copysize"))
|
||||
}
|
||||
|
||||
func (s *PublishedStorageSuite) TestMkDir(c *C) {
|
||||
@@ -35,7 +56,7 @@ func (s *PublishedStorageSuite) TestMkDir(c *C) {
|
||||
c.Assert(err, IsNil)
|
||||
}
|
||||
|
||||
func (s *PublishedStorageSuite) TesPutFile(c *C) {
|
||||
func (s *PublishedStorageSuite) TestPutFile(c *C) {
|
||||
err := s.storage.MkDir("ppa/dists/squeeze/")
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
@@ -156,7 +177,10 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
|
||||
err = ioutil.WriteFile(t.sourcePath, []byte("Contents"), 0644)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, utils.ChecksumInfo{}, false)
|
||||
sourceChecksum, err := utils.ChecksumsForFile(t.sourcePath)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, sourceChecksum, false)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename))
|
||||
@@ -164,6 +188,36 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
|
||||
|
||||
info := st.Sys().(*syscall.Stat_t)
|
||||
c.Check(int(info.Nlink), Equals, 2)
|
||||
|
||||
// Test using symlinks
|
||||
err = s.storageSymlink.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, sourceChecksum, false)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
st, err = os.Stat(filepath.Join(s.storageSymlink.rootPath, t.prefix, t.expectedFilename))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
info = st.Sys().(*syscall.Stat_t)
|
||||
c.Check(int(info.Nlink), Equals, 2)
|
||||
|
||||
// Test using copy with checksum verification
|
||||
err = s.storageCopy.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, sourceChecksum, false)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
st, err = os.Stat(filepath.Join(s.storageCopy.rootPath, t.prefix, t.expectedFilename))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
info = st.Sys().(*syscall.Stat_t)
|
||||
c.Check(int(info.Nlink), Equals, 1)
|
||||
|
||||
// Test using copy with size verification
|
||||
err = s.storageCopySize.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, sourceChecksum, false)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
st, err = os.Stat(filepath.Join(s.storageCopySize.rootPath, t.prefix, t.expectedFilename))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
info = st.Sys().(*syscall.Stat_t)
|
||||
c.Check(int(info.Nlink), Equals, 1)
|
||||
}
|
||||
|
||||
// test linking files to duplicate final name
|
||||
@@ -171,10 +225,14 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
|
||||
err := os.MkdirAll(filepath.Dir(sourcePath), 0755)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
err = ioutil.WriteFile(sourcePath, []byte("Contents"), 0644)
|
||||
// use same size to ensure copy with size check will fail on this one
|
||||
err = ioutil.WriteFile(sourcePath, []byte("cONTENTS"), 0644)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, utils.ChecksumInfo{}, false)
|
||||
sourceChecksum, err := utils.ChecksumsForFile(sourcePath)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, false)
|
||||
c.Check(err, ErrorMatches, ".*file already exists and is different")
|
||||
|
||||
st, err := os.Stat(sourcePath)
|
||||
@@ -184,7 +242,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
|
||||
c.Check(int(info.Nlink), Equals, 1)
|
||||
|
||||
// linking with force
|
||||
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, utils.ChecksumInfo{}, true)
|
||||
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, true)
|
||||
c.Check(err, IsNil)
|
||||
|
||||
st, err = os.Stat(sourcePath)
|
||||
@@ -192,4 +250,22 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
|
||||
|
||||
info = st.Sys().(*syscall.Stat_t)
|
||||
c.Check(int(info.Nlink), Equals, 2)
|
||||
|
||||
// Test using symlinks
|
||||
err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, false)
|
||||
c.Check(err, ErrorMatches, ".*file already exists and is different")
|
||||
|
||||
err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, true)
|
||||
c.Check(err, IsNil)
|
||||
|
||||
// Test using copy with checksum verification
|
||||
err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, false)
|
||||
c.Check(err, ErrorMatches, ".*file already exists and is different")
|
||||
|
||||
err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, true)
|
||||
c.Check(err, IsNil)
|
||||
|
||||
// Test using copy with size verification (this will NOT detect the difference)
|
||||
err = s.storageCopySize.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, sourceChecksum, false)
|
||||
c.Check(err, IsNil)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user