Imported Upstream version 1.0.1

This commit is contained in:
Sébastien Delafond
2017-07-04 14:45:08 +02:00
parent 1ee35b841d
commit 4f12259bc0
5531 changed files with 1834599 additions and 742844 deletions
+19
View File
@@ -0,0 +1,19 @@
// Package files handles operation on filesystem for both public pool and published files
package files
// Repository directory structure:
// <root>
// \- pool
// \- ab
// \- ae
// \- package.deb
// \- public
// \- dists
// \- squeeze
// \- Release
// \- main
// \- binary-i386
// \- Packages.bz2
// references packages from pool
// \- pool
// contains symlinks to main pool
+12
View File
@@ -0,0 +1,12 @@
package files
import (
"testing"
. "gopkg.in/check.v1"
)
// Launch gocheck tests
func Test(t *testing.T) {
TestingT(t)
}
+169
View File
@@ -0,0 +1,169 @@
package files
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sync"
"github.com/smira/aptly/aptly"
)
// PackagePool is deduplicated storage of package files on filesystem
type PackagePool struct {
sync.Mutex
rootPath string
}
// Check interface
var (
_ aptly.PackagePool = (*PackagePool)(nil)
)
// NewPackagePool creates new instance of PackagePool which specified root
func NewPackagePool(root string) *PackagePool {
return &PackagePool{rootPath: filepath.Join(root, "pool")}
}
// RelativePath returns path relative to pool's root for package files given MD5 and original filename
func (pool *PackagePool) RelativePath(filename string, hashMD5 string) (string, error) {
filename = filepath.Base(filename)
if filename == "." || filename == "/" {
return "", fmt.Errorf("filename %s is invalid", filename)
}
if len(hashMD5) < 4 {
return "", fmt.Errorf("unable to compute pool location for filename %v, MD5 is missing", filename)
}
return filepath.Join(hashMD5[0:2], hashMD5[2:4], filename), nil
}
// Path returns full path to package file in pool given any name and hash of file contents
func (pool *PackagePool) Path(filename string, hashMD5 string) (string, error) {
relative, err := pool.RelativePath(filename, hashMD5)
if err != nil {
return "", err
}
return filepath.Join(pool.rootPath, relative), nil
}
// FilepathList returns file paths of all the files in the pool
func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) {
pool.Lock()
defer pool.Unlock()
dirs, err := ioutil.ReadDir(pool.rootPath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
if len(dirs) == 0 {
return nil, nil
}
if progress != nil {
progress.InitBar(int64(len(dirs)), false)
defer progress.ShutdownBar()
}
result := []string{}
for _, dir := range dirs {
err = filepath.Walk(filepath.Join(pool.rootPath, dir.Name()), func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
result = append(result, path[len(pool.rootPath)+1:])
}
return nil
})
if err != nil {
return nil, err
}
if progress != nil {
progress.AddBar(1)
}
}
return result, nil
}
// Remove deletes file in package pool returns its size
func (pool *PackagePool) Remove(path string) (size int64, err error) {
pool.Lock()
defer pool.Unlock()
path = filepath.Join(pool.rootPath, path)
info, err := os.Stat(path)
if err != nil {
return 0, err
}
err = os.Remove(path)
return info.Size(), err
}
// Import copies file into package pool
func (pool *PackagePool) Import(path string, hashMD5 string) error {
pool.Lock()
defer pool.Unlock()
source, err := os.Open(path)
if err != nil {
return err
}
defer source.Close()
sourceInfo, err := source.Stat()
if err != nil {
return err
}
poolPath, err := pool.Path(path, hashMD5)
if err != nil {
return err
}
targetInfo, err := os.Stat(poolPath)
if err != nil {
if !os.IsNotExist(err) {
// unable to stat target location?
return err
}
} else {
// target already exists
if targetInfo.Size() != sourceInfo.Size() {
// trying to overwrite file?
return fmt.Errorf("unable to import into pool: file %s already exists", poolPath)
}
// assume that target is already there
return nil
}
// create subdirs as necessary
err = os.MkdirAll(filepath.Dir(poolPath), 0777)
if err != nil {
return err
}
target, err := os.Create(poolPath)
if err != nil {
return err
}
defer target.Close()
_, err = io.Copy(target, source)
return err
}
+120
View File
@@ -0,0 +1,120 @@
package files
import (
"io/ioutil"
"os"
"path/filepath"
"runtime"
. "gopkg.in/check.v1"
)
type PackagePoolSuite struct {
pool *PackagePool
}
var _ = Suite(&PackagePoolSuite{})
func (s *PackagePoolSuite) SetUpTest(c *C) {
s.pool = NewPackagePool(c.MkDir())
}
func (s *PackagePoolSuite) TestRelativePath(c *C) {
path, err := s.pool.RelativePath("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, IsNil)
c.Assert(path, Equals, "91/b1/package.deb")
_, err = s.pool.RelativePath("/", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, ErrorMatches, ".*is invalid")
_, err = s.pool.RelativePath("", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, ErrorMatches, ".*is invalid")
_, err = s.pool.RelativePath("a/b/package.deb", "9")
c.Assert(err, ErrorMatches, ".*MD5 is missing")
}
func (s *PackagePoolSuite) TestPath(c *C) {
path, err := s.pool.Path("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, IsNil)
c.Assert(path, Equals, filepath.Join(s.pool.rootPath, "91/b1/package.deb"))
_, err = s.pool.Path("/", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, ErrorMatches, ".*is invalid")
}
func (s *PackagePoolSuite) TestFilepathList(c *C) {
list, err := s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, IsNil)
os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0b"), 0755)
os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0a"), 0755)
os.MkdirAll(filepath.Join(s.pool.rootPath, "ae", "0c"), 0755)
list, err = s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{})
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "1.deb"), nil, 0644)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "2.deb"), nil, 0644)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0a", "3.deb"), nil, 0644)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0b", "4.deb"), nil, 0644)
list, err = s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"ae/0c/1.deb", "ae/0c/2.deb", "bd/0a/3.deb", "bd/0b/4.deb"})
}
func (s *PackagePoolSuite) TestRemove(c *C) {
os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0b"), 0755)
os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0a"), 0755)
os.MkdirAll(filepath.Join(s.pool.rootPath, "ae", "0c"), 0755)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "1.deb"), []byte("1"), 0644)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "2.deb"), []byte("22"), 0644)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0a", "3.deb"), []byte("333"), 0644)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0b", "4.deb"), []byte("4444"), 0644)
size, err := s.pool.Remove("ae/0c/2.deb")
c.Check(err, IsNil)
c.Check(size, Equals, int64(2))
_, err = s.pool.Remove("ae/0c/2.deb")
c.Check(err, ErrorMatches, ".*no such file or directory")
list, err := s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"ae/0c/1.deb", "bd/0a/3.deb", "bd/0b/4.deb"})
}
func (s *PackagePoolSuite) TestImportOk(c *C) {
_, _File, _, _ := runtime.Caller(0)
debFile := filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
err := s.pool.Import(debFile, "91b1a1480b90b9e269ca44d897b12575")
c.Check(err, IsNil)
info, err := os.Stat(filepath.Join(s.pool.rootPath, "91", "b1", "libboost-program-options-dev_1.49.0.1_i386.deb"))
c.Check(err, IsNil)
c.Check(info.Size(), Equals, int64(2738))
// double import, should be ok
err = s.pool.Import(debFile, "91b1a1480b90b9e269ca44d897b12575")
c.Check(err, IsNil)
}
func (s *PackagePoolSuite) TestImportNotExist(c *C) {
err := s.pool.Import("no-such-file", "91b1a1480b90b9e269ca44d897b12575")
c.Check(err, ErrorMatches, ".*no such file or directory")
}
func (s *PackagePoolSuite) TestImportOverwrite(c *C) {
_, _File, _, _ := runtime.Caller(0)
debFile := filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
os.MkdirAll(filepath.Join(s.pool.rootPath, "91", "b1"), 0755)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "91", "b1", "libboost-program-options-dev_1.49.0.1_i386.deb"), []byte("1"), 0644)
err := s.pool.Import(debFile, "91b1a1480b90b9e269ca44d897b12575")
c.Check(err, ErrorMatches, "unable to import into pool.*")
}
+157
View File
@@ -0,0 +1,157 @@
package files
import (
"fmt"
"io"
"os"
"path/filepath"
"syscall"
"github.com/smira/aptly/aptly"
)
// PublishedStorage abstract file system with public dirs (published repos)
type PublishedStorage struct {
rootPath string
}
// Check interfaces
var (
_ aptly.PublishedStorage = (*PublishedStorage)(nil)
_ aptly.LocalPublishedStorage = (*PublishedStorage)(nil)
)
// NewPublishedStorage creates new instance of PublishedStorage which specified root
func NewPublishedStorage(root string) *PublishedStorage {
return &PublishedStorage{rootPath: filepath.Join(root, "public")}
}
// PublicPath returns root of public part
func (storage *PublishedStorage) PublicPath() string {
return storage.rootPath
}
// MkDir creates directory recursively under public path
func (storage *PublishedStorage) MkDir(path string) error {
return os.MkdirAll(filepath.Join(storage.rootPath, path), 0777)
}
// PutFile puts file into published storage at specified path
func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
var (
source, f *os.File
err error
)
source, err = os.Open(sourceFilename)
if err != nil {
return err
}
defer source.Close()
f, err = os.Create(filepath.Join(storage.rootPath, path))
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(f, source)
return err
}
// Remove removes single file under public path
func (storage *PublishedStorage) Remove(path string) error {
filepath := filepath.Join(storage.rootPath, path)
return os.Remove(filepath)
}
// RemoveDirs removes directory structure under public path
func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
filepath := filepath.Join(storage.rootPath, path)
if progress != nil {
progress.Printf("Removing %s...\n", filepath)
}
return os.RemoveAll(filepath)
}
// LinkFromPool links package file from pool to dist's pool location
//
// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
// sourcePool is instance of aptly.PackagePool
// sourcePath is filepath to package file in package pool
//
// LinkFromPool returns relative path for the published file to be included in package index
func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool,
sourcePath, sourceMD5 string, force bool) error {
// verify that package pool is local pool is filesystem pool
_ = sourcePool.(*PackagePool)
baseName := filepath.Base(sourcePath)
poolPath := filepath.Join(storage.rootPath, publishedDirectory)
err := os.MkdirAll(poolPath, 0777)
if err != nil {
return err
}
var dstStat, srcStat os.FileInfo
dstStat, err = os.Stat(filepath.Join(poolPath, baseName))
if err == nil {
// already exists, check source file
srcStat, err = os.Stat(sourcePath)
if err != nil {
// source file doesn't exist? problem!
return err
}
srcSys := srcStat.Sys().(*syscall.Stat_t)
dstSys := dstStat.Sys().(*syscall.Stat_t)
// source and destination inodes match, no need to link
if srcSys.Ino == dstSys.Ino {
return nil
}
// source and destination have different inodes, if !forced, this is fatal error
if !force {
return fmt.Errorf("error linking file to %s: file already exists and is different", filepath.Join(poolPath, baseName))
}
// forced, so remove destination
err = os.Remove(filepath.Join(poolPath, baseName))
if err != nil {
return err
}
}
// destination doesn't exist (or forced), create link
return os.Link(sourcePath, filepath.Join(poolPath, baseName))
}
// Filelist returns list of files under prefix
func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
root := filepath.Join(storage.rootPath, prefix)
result := []string{}
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
result = append(result, path[len(root)+1:])
}
return nil
})
if err != nil && os.IsNotExist(err) {
// file path doesn't exist, consider it empty
return []string{}, nil
}
return result, err
}
// RenameFile renames (moves) file
func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
return os.Rename(filepath.Join(storage.rootPath, oldName), filepath.Join(storage.rootPath, newName))
}
+193
View File
@@ -0,0 +1,193 @@
package files
import (
"io/ioutil"
"os"
"path/filepath"
"syscall"
. "gopkg.in/check.v1"
)
type PublishedStorageSuite struct {
root string
storage *PublishedStorage
}
var _ = Suite(&PublishedStorageSuite{})
func (s *PublishedStorageSuite) SetUpTest(c *C) {
s.root = c.MkDir()
s.storage = NewPublishedStorage(s.root)
}
func (s *PublishedStorageSuite) TestPublicPath(c *C) {
c.Assert(s.storage.PublicPath(), Equals, filepath.Join(s.root, "public"))
}
func (s *PublishedStorageSuite) TestMkDir(c *C) {
err := s.storage.MkDir("ppa/dists/squeeze/")
c.Assert(err, IsNil)
_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/"))
c.Assert(err, IsNil)
}
func (s *PublishedStorageSuite) TesPutFile(c *C) {
err := s.storage.MkDir("ppa/dists/squeeze/")
c.Assert(err, IsNil)
err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
c.Assert(err, IsNil)
_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
c.Assert(err, IsNil)
}
func (s *PublishedStorageSuite) TestFilelist(c *C) {
err := s.storage.MkDir("ppa/pool/main/a/ab/")
c.Assert(err, IsNil)
err = s.storage.PutFile("ppa/pool/main/a/ab/a.deb", "/dev/null")
c.Assert(err, IsNil)
err = s.storage.PutFile("ppa/pool/main/a/ab/b.deb", "/dev/null")
c.Assert(err, IsNil)
list, err := s.storage.Filelist("ppa/pool/main/")
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"a/ab/a.deb", "a/ab/b.deb"})
list, err = s.storage.Filelist("ppa/pool/doenstexist/")
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{})
}
func (s *PublishedStorageSuite) TestRenameFile(c *C) {
err := s.storage.MkDir("ppa/dists/squeeze/")
c.Assert(err, IsNil)
err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
c.Assert(err, IsNil)
err = s.storage.RenameFile("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease")
c.Check(err, IsNil)
_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/InRelease"))
c.Assert(err, IsNil)
}
func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
err := s.storage.MkDir("ppa/dists/squeeze/")
c.Assert(err, IsNil)
err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
c.Assert(err, IsNil)
err = s.storage.RemoveDirs("ppa/dists/", nil)
_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
c.Assert(err, NotNil)
c.Assert(os.IsNotExist(err), Equals, true)
}
func (s *PublishedStorageSuite) TestRemove(c *C) {
err := s.storage.MkDir("ppa/dists/squeeze/")
c.Assert(err, IsNil)
err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
c.Assert(err, IsNil)
err = s.storage.Remove("ppa/dists/squeeze/Release")
_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
c.Assert(err, NotNil)
c.Assert(os.IsNotExist(err), Equals, true)
}
func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
tests := []struct {
prefix string
component string
sourcePath string
poolDirectory string
expectedFilename string
}{
{ // package name regular
prefix: "",
component: "main",
sourcePath: "pool/01/ae/mars-invaders_1.03.deb",
poolDirectory: "m/mars-invaders",
expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
},
{ // lib-like filename
prefix: "",
component: "main",
sourcePath: "pool/01/ae/libmars-invaders_1.03.deb",
poolDirectory: "libm/libmars-invaders",
expectedFilename: "pool/main/libm/libmars-invaders/libmars-invaders_1.03.deb",
},
{ // duplicate link, shouldn't panic
prefix: "",
component: "main",
sourcePath: "pool/01/ae/mars-invaders_1.03.deb",
poolDirectory: "m/mars-invaders",
expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
},
{ // prefix & component
prefix: "ppa",
component: "contrib",
sourcePath: "pool/01/ae/libmars-invaders_1.04.deb",
poolDirectory: "libm/libmars-invaders",
expectedFilename: "pool/contrib/libm/libmars-invaders/libmars-invaders_1.04.deb",
},
}
pool := NewPackagePool(s.root)
for _, t := range tests {
t.sourcePath = filepath.Join(s.root, t.sourcePath)
err := os.MkdirAll(filepath.Dir(t.sourcePath), 0755)
c.Assert(err, IsNil)
err = ioutil.WriteFile(t.sourcePath, []byte("Contents"), 0644)
c.Assert(err, IsNil)
err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, "", false)
c.Assert(err, IsNil)
st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename))
c.Assert(err, IsNil)
info := st.Sys().(*syscall.Stat_t)
c.Check(int(info.Nlink), Equals, 2)
}
// test linking files to duplicate final name
sourcePath := filepath.Join(s.root, "pool/02/bc/mars-invaders_1.03.deb")
err := os.MkdirAll(filepath.Dir(sourcePath), 0755)
c.Assert(err, IsNil)
err = ioutil.WriteFile(sourcePath, []byte("Contents"), 0644)
c.Assert(err, IsNil)
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "", false)
c.Check(err, ErrorMatches, ".*file already exists and is different")
st, err := os.Stat(sourcePath)
c.Assert(err, IsNil)
info := st.Sys().(*syscall.Stat_t)
c.Check(int(info.Nlink), Equals, 1)
// linking with force
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "", true)
c.Check(err, IsNil)
st, err = os.Stat(sourcePath)
c.Assert(err, IsNil)
info = st.Sys().(*syscall.Stat_t)
c.Check(int(info.Nlink), Equals, 2)
}