mirror of
https://github.com/aptly-dev/aptly.git
synced 2026-06-02 04:50:49 +00:00
Refactor Repository: split into PackagePool and PublishedStorage.
This commit is contained in:
Andrey Smirnov
@@ -0,0 +1,19 @@
|
||||
// Package files handles operation on filesystem for both public pool and published files
|
||||
package files
|
||||
|
||||
// Repository directory structure:
|
||||
// <root>
|
||||
// \- pool
|
||||
// \- ab
|
||||
// \- ae
|
||||
// \- package.deb
|
||||
// \- public
|
||||
// \- dists
|
||||
// \- squeeze
|
||||
// \- Release
|
||||
// \- main
|
||||
// \- binary-i386
|
||||
// \- Packages.bz2
|
||||
// references packages from pool
|
||||
// \- pool
|
||||
// contains symlinks to main pool
|
||||
@@ -0,0 +1,11 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
. "launchpad.net/gocheck"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Launch gocheck tests
|
||||
func Test(t *testing.T) {
|
||||
TestingT(t)
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/smira/aptly/aptly"
|
||||
"github.com/smira/aptly/utils"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// PackagePool is deduplicated storage of package files on filesystem
|
||||
type PackagePool struct {
|
||||
rootPath string
|
||||
}
|
||||
|
||||
// Check interface
|
||||
var (
|
||||
_ aptly.PackagePool = (*PackagePool)(nil)
|
||||
)
|
||||
|
||||
// NewPackagePool creates new instance of PackagePool which specified root
|
||||
func NewPackagePool(root string) *PackagePool {
|
||||
return &PackagePool{rootPath: filepath.Join(root, "pool")}
|
||||
}
|
||||
|
||||
// RelativePath returns path relative to pool's root for package files given MD5 and original filename
|
||||
func (pool *PackagePool) RelativePath(filename string, hashMD5 string) (string, error) {
|
||||
filename = filepath.Base(filename)
|
||||
if filename == "." || filename == "/" {
|
||||
return "", fmt.Errorf("filename %s is invalid", filename)
|
||||
}
|
||||
|
||||
if len(hashMD5) < 4 {
|
||||
return "", fmt.Errorf("unable to compute pool location for filename %v, MD5 is missing", filename)
|
||||
}
|
||||
|
||||
return filepath.Join(hashMD5[0:2], hashMD5[2:4], filename), nil
|
||||
}
|
||||
|
||||
// Path returns full path to package file in pool given any name and hash of file contents
|
||||
func (p *PackagePool) Path(filename string, hashMD5 string) (string, error) {
|
||||
relative, err := p.RelativePath(filename, hashMD5)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return filepath.Join(p.rootPath, relative), nil
|
||||
}
|
||||
|
||||
// FilepathList returns file paths of all the files in the pool
|
||||
func (p *PackagePool) FilepathList(progress *utils.Progress) ([]string, error) {
|
||||
dirs, err := ioutil.ReadDir(p.rootPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(dirs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if progress != nil {
|
||||
progress.InitBar(int64(len(dirs)), false)
|
||||
defer progress.ShutdownBar()
|
||||
}
|
||||
|
||||
result := []string{}
|
||||
|
||||
for _, dir := range dirs {
|
||||
err = filepath.Walk(filepath.Join(p.rootPath, dir.Name()), func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
result = append(result, path[len(p.rootPath)+1:])
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if progress != nil {
|
||||
progress.AddBar(1)
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Remove deletes file in package pool returns its size
|
||||
func (p *PackagePool) Remove(path string) (size int64, err error) {
|
||||
path = filepath.Join(p.rootPath, path)
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
err = os.Remove(path)
|
||||
return info.Size(), err
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
. "launchpad.net/gocheck"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type PackagePoolSuite struct {
|
||||
pool *PackagePool
|
||||
}
|
||||
|
||||
var _ = Suite(&PackagePoolSuite{})
|
||||
|
||||
func (s *PackagePoolSuite) SetUpTest(c *C) {
|
||||
s.pool = NewPackagePool(c.MkDir())
|
||||
|
||||
}
|
||||
|
||||
func (s *PackagePoolSuite) TestRelativePath(c *C) {
|
||||
path, err := s.pool.RelativePath("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(path, Equals, "91/b1/package.deb")
|
||||
|
||||
_, err = s.pool.RelativePath("/", "91b1a1480b90b9e269ca44d897b12575")
|
||||
c.Assert(err, ErrorMatches, ".*is invalid")
|
||||
_, err = s.pool.RelativePath("", "91b1a1480b90b9e269ca44d897b12575")
|
||||
c.Assert(err, ErrorMatches, ".*is invalid")
|
||||
_, err = s.pool.RelativePath("a/b/package.deb", "9")
|
||||
c.Assert(err, ErrorMatches, ".*MD5 is missing")
|
||||
}
|
||||
|
||||
func (s *PackagePoolSuite) TestPath(c *C) {
|
||||
path, err := s.pool.Path("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(path, Equals, filepath.Join(s.pool.rootPath, "91/b1/package.deb"))
|
||||
|
||||
_, err = s.pool.Path("/", "91b1a1480b90b9e269ca44d897b12575")
|
||||
c.Assert(err, ErrorMatches, ".*is invalid")
|
||||
}
|
||||
|
||||
func (s *PackagePoolSuite) TestFilepathList(c *C) {
|
||||
list, err := s.pool.FilepathList(nil)
|
||||
c.Check(err, IsNil)
|
||||
c.Check(list, IsNil)
|
||||
|
||||
os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0b"), 0755)
|
||||
os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0a"), 0755)
|
||||
os.MkdirAll(filepath.Join(s.pool.rootPath, "ae", "0c"), 0755)
|
||||
|
||||
list, err = s.pool.FilepathList(nil)
|
||||
c.Check(err, IsNil)
|
||||
c.Check(list, DeepEquals, []string{})
|
||||
|
||||
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "1.deb"), nil, 0644)
|
||||
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "2.deb"), nil, 0644)
|
||||
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0a", "3.deb"), nil, 0644)
|
||||
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0b", "4.deb"), nil, 0644)
|
||||
|
||||
list, err = s.pool.FilepathList(nil)
|
||||
c.Check(err, IsNil)
|
||||
c.Check(list, DeepEquals, []string{"ae/0c/1.deb", "ae/0c/2.deb", "bd/0a/3.deb", "bd/0b/4.deb"})
|
||||
}
|
||||
|
||||
func (s *PackagePoolSuite) TestRemove(c *C) {
|
||||
os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0b"), 0755)
|
||||
os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0a"), 0755)
|
||||
os.MkdirAll(filepath.Join(s.pool.rootPath, "ae", "0c"), 0755)
|
||||
|
||||
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "1.deb"), []byte("1"), 0644)
|
||||
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "2.deb"), []byte("22"), 0644)
|
||||
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0a", "3.deb"), []byte("333"), 0644)
|
||||
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0b", "4.deb"), []byte("4444"), 0644)
|
||||
|
||||
size, err := s.pool.Remove("ae/0c/2.deb")
|
||||
c.Check(err, IsNil)
|
||||
c.Check(size, Equals, int64(2))
|
||||
|
||||
_, err = s.pool.Remove("ae/0c/2.deb")
|
||||
c.Check(err, ErrorMatches, ".*no such file or directory")
|
||||
|
||||
list, err := s.pool.FilepathList(nil)
|
||||
c.Check(err, IsNil)
|
||||
c.Check(list, DeepEquals, []string{"ae/0c/1.deb", "bd/0a/3.deb", "bd/0b/4.deb"})
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/smira/aptly/aptly"
|
||||
"github.com/smira/aptly/utils"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// PublishedStorage abstract file system with public dirs (published repos)
|
||||
type PublishedStorage struct {
|
||||
rootPath string
|
||||
}
|
||||
|
||||
// Check interface
|
||||
var (
|
||||
_ aptly.PublishedStorage = (*PublishedStorage)(nil)
|
||||
)
|
||||
|
||||
// NewPublishedStorage creates new instance of PublishedStorage which specified root
|
||||
func NewPublishedStorage(root string) *PublishedStorage {
|
||||
return &PublishedStorage{rootPath: filepath.Join(root, "public")}
|
||||
}
|
||||
|
||||
// PublicPath returns root of public part
|
||||
func (storage *PublishedStorage) PublicPath() string {
|
||||
return storage.rootPath
|
||||
}
|
||||
|
||||
// MkDir creates directory recursively under public path
|
||||
func (storage *PublishedStorage) MkDir(path string) error {
|
||||
return os.MkdirAll(filepath.Join(storage.rootPath, path), 0755)
|
||||
}
|
||||
|
||||
// CreateFile creates file for writing under public path
|
||||
func (storage *PublishedStorage) CreateFile(path string) (*os.File, error) {
|
||||
return os.Create(filepath.Join(storage.rootPath, path))
|
||||
}
|
||||
|
||||
// RemoveDirs removes directory structure under public path
|
||||
func (storage *PublishedStorage) RemoveDirs(path string) error {
|
||||
filepath := filepath.Join(storage.rootPath, path)
|
||||
fmt.Printf("Removing %s...\n", filepath)
|
||||
return os.RemoveAll(filepath)
|
||||
}
|
||||
|
||||
// LinkFromPool links package file from pool to dist's pool location
|
||||
//
|
||||
// prefix is publishing prefix for this repo (e.g. empty or "ppa/")
|
||||
// component is component name when publishing (e.g. main)
|
||||
// poolDirectory is desired location in pool (like liba/libav/)
|
||||
// sourcePool is instance of aptly.PackagePool
|
||||
// sourcePath is filepath to package file in package pool
|
||||
//
|
||||
// LinkFromPool returns relative path for the published file to be included in package index
|
||||
func (storage *PublishedStorage) LinkFromPool(prefix string, component string, poolDirectory string, sourcePool aptly.PackagePool, sourcePath string) (string, error) {
|
||||
// verify that package pool is local pool is filesystem pool
|
||||
_ = sourcePool.(*PackagePool)
|
||||
|
||||
baseName := filepath.Base(sourcePath)
|
||||
|
||||
relPath := filepath.Join("pool", component, poolDirectory, baseName)
|
||||
poolPath := filepath.Join(storage.rootPath, prefix, "pool", component, poolDirectory)
|
||||
|
||||
err := os.MkdirAll(poolPath, 0755)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
_, err = os.Stat(filepath.Join(poolPath, baseName))
|
||||
if err == nil { // already exists, skip
|
||||
return relPath, nil
|
||||
}
|
||||
|
||||
err = os.Link(sourcePath, filepath.Join(poolPath, baseName))
|
||||
return relPath, err
|
||||
}
|
||||
|
||||
// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
|
||||
func (storage *PublishedStorage) ChecksumsForFile(path string) (utils.ChecksumInfo, error) {
|
||||
return utils.ChecksumsForFile(filepath.Join(storage.rootPath, path))
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
. "launchpad.net/gocheck"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
type PublishedStorageSuite struct {
|
||||
root string
|
||||
storage *PublishedStorage
|
||||
}
|
||||
|
||||
var _ = Suite(&PublishedStorageSuite{})
|
||||
|
||||
func (s *PublishedStorageSuite) SetUpTest(c *C) {
|
||||
s.root = c.MkDir()
|
||||
s.storage = NewPublishedStorage(s.root)
|
||||
}
|
||||
|
||||
func (s *PublishedStorageSuite) TestPublicPath(c *C) {
|
||||
c.Assert(s.storage.PublicPath(), Equals, filepath.Join(s.root, "public"))
|
||||
}
|
||||
|
||||
func (s *PublishedStorageSuite) TestMkDir(c *C) {
|
||||
err := s.storage.MkDir("ppa/dists/squeeze/")
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/"))
|
||||
c.Assert(err, IsNil)
|
||||
}
|
||||
|
||||
func (s *PublishedStorageSuite) TestCreateFile(c *C) {
|
||||
err := s.storage.MkDir("ppa/dists/squeeze/")
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
|
||||
c.Assert(err, IsNil)
|
||||
defer file.Close()
|
||||
|
||||
_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
|
||||
c.Assert(err, IsNil)
|
||||
}
|
||||
|
||||
func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
|
||||
err := s.storage.MkDir("ppa/dists/squeeze/")
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
|
||||
c.Assert(err, IsNil)
|
||||
defer file.Close()
|
||||
|
||||
err = s.storage.RemoveDirs("ppa/dists/")
|
||||
|
||||
_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
|
||||
c.Assert(err, NotNil)
|
||||
c.Assert(os.IsNotExist(err), Equals, true)
|
||||
}
|
||||
|
||||
func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
|
||||
tests := []struct {
|
||||
prefix string
|
||||
component string
|
||||
sourcePath string
|
||||
poolDirectory string
|
||||
expectedFilename string
|
||||
}{
|
||||
{ // package name regular
|
||||
prefix: "",
|
||||
component: "main",
|
||||
sourcePath: "pool/01/ae/mars-invaders_1.03.deb",
|
||||
poolDirectory: "m/mars-invaders",
|
||||
expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
|
||||
},
|
||||
{ // lib-like filename
|
||||
prefix: "",
|
||||
component: "main",
|
||||
sourcePath: "pool/01/ae/libmars-invaders_1.03.deb",
|
||||
poolDirectory: "libm/libmars-invaders",
|
||||
expectedFilename: "pool/main/libm/libmars-invaders/libmars-invaders_1.03.deb",
|
||||
},
|
||||
{ // duplicate link, shouldn't panic
|
||||
prefix: "",
|
||||
component: "main",
|
||||
sourcePath: "pool/01/ae/mars-invaders_1.03.deb",
|
||||
poolDirectory: "m/mars-invaders",
|
||||
expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
|
||||
},
|
||||
{ // prefix & component
|
||||
prefix: "ppa",
|
||||
component: "contrib",
|
||||
sourcePath: "pool/01/ae/libmars-invaders_1.04.deb",
|
||||
poolDirectory: "libm/libmars-invaders",
|
||||
expectedFilename: "pool/contrib/libm/libmars-invaders/libmars-invaders_1.04.deb",
|
||||
},
|
||||
}
|
||||
|
||||
pool := NewPackagePool(s.root)
|
||||
|
||||
for _, t := range tests {
|
||||
t.sourcePath = filepath.Join(s.root, t.sourcePath)
|
||||
|
||||
err := os.MkdirAll(filepath.Dir(t.sourcePath), 0755)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
err = ioutil.WriteFile(t.sourcePath, []byte("Contents"), 0644)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
path, err := s.storage.LinkFromPool(t.prefix, t.component, t.poolDirectory, pool, t.sourcePath)
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(path, Equals, t.expectedFilename)
|
||||
|
||||
st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
info := st.Sys().(*syscall.Stat_t)
|
||||
c.Check(int(info.Nlink), Equals, 2)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user