vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-01-12 03:21:33 +00:00
Code Issues Projects Releases Wiki Activity

Refactor Repository: split into PackagePool and PublishedStorage.

Browse Source
This commit is contained in:
Andrey Smirnov
2014-02-19 12:03:01 +04:00
parent 7864ce241b
commit d1e16a0ef0
20 changed files with 616 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
aptly/interfaces.go Normal file
View File

@@ -0,0 +1,28 @@
// Package aptly provides common infrastructure that doesn't depend directly on
// Debian or CentOS
package aptly
import (
"github.com/smira/aptly/utils"
"os"
)
// PackagePool is asbtraction of package pool storage.
//
// PackagePool stores all the package files, deduplicating them.
type PackagePool interface {
Path(filename string, hashMD5 string) (string, error)
RelativePath(filename string, hashMD5 string) (string, error)
FilepathList(progress *utils.Progress) ([]string, error)
Remove(path string) (size int64, err error)
}
// PublishedStorage is abstraction of filesystem storing all published repositories
type PublishedStorage interface {
PublicPath() string
MkDir(path string) error
CreateFile(path string) (*os.File, error)
RemoveDirs(path string) error
LinkFromPool(prefix string, component string, poolDirectory string, sourcePool PackagePool, sourcePath string) (string, error)
ChecksumsForFile(path string) (utils.ChecksumInfo, error)
}

6
cmd_db_cleanup.go
View File

@@ -80,7 +80,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
if err != nil {
return err
}
paths, err := pkg.FilepathList(context.packageRepository)
paths, err := pkg.FilepathList(context.packagePool)
if err != nil {
return err
}
@@ -98,7 +98,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
// build a list of files in the package pool
context.downloader.GetProgress().Printf("Building list of files in package pool...\n")
existingFiles, err := context.packageRepository.PoolFilepathList(context.downloader.GetProgress())
existingFiles, err := context.packagePool.FilepathList(context.downloader.GetProgress())
if err != nil {
return fmt.Errorf("unable to collect file paths: %s", err)
}
@@ -113,7 +113,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
context.downloader.GetProgress().InitBar(int64(len(filesToDelete)), false)
totalSize := int64(0)
for _, file := range filesToDelete {
size, err := context.packageRepository.PoolRemove(file)
size, err := context.packagePool.Remove(file)
if err != nil {
return err
}

2
cmd_mirror.go
View File

@@ -201,7 +201,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
packageCollection := debian.NewPackageCollection(context.database)
err = repo.Download(context.downloader, packageCollection, context.packageRepository, ignoreMismatch)
err = repo.Download(context.downloader, packageCollection, context.packagePool, ignoreMismatch)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}

6
cmd_publish.go
View File

@@ -102,7 +102,7 @@ func aptlyPublishSnapshot(cmd *commander.Command, args []string) error {
}
packageCollection := debian.NewPackageCollection(context.database)
err = published.Publish(context.packageRepository, packageCollection, signer)
err = published.Publish(context.packagePool, context.publishedStorage, packageCollection, signer)
if err != nil {
return fmt.Errorf("unable to publish: %s", err)
}
@@ -117,7 +117,7 @@ func aptlyPublishSnapshot(cmd *commander.Command, args []string) error {
}
fmt.Printf("\nSnapshot %s has been successfully published.\nPlease setup your webserver to serve directory '%s' with autoindexing.\n",
snapshot.Name, context.packageRepository.PublicPath())
snapshot.Name, context.publishedStorage.PublicPath())
fmt.Printf("Now you can add following line to apt sources:\n")
fmt.Printf(" deb http://your-server/%s %s %s\n", prefix, distribution, component)
fmt.Printf("Don't forget to add your GPG key to apt with apt-key.\n")
@@ -184,7 +184,7 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
publishedCollecton := debian.NewPublishedRepoCollection(context.database)
err = publishedCollecton.Remove(context.packageRepository, prefix, distribution)
err = publishedCollecton.Remove(context.publishedStorage, prefix, distribution)
if err != nil {
return fmt.Errorf("unable to remove: %s", err)
}

2
cmd_serve.go
View File

@@ -84,7 +84,7 @@ func aptlyServe(cmd *commander.Command, args []string) error {
fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
err = http.ListenAndServe(listen, http.FileServer(http.Dir(context.packageRepository.PublicPath())))
err = http.ListenAndServe(listen, http.FileServer(http.Dir(context.publishedStorage.PublicPath())))
if err != nil {
return fmt.Errorf("unable to serve: %s", err)
}

25
debian/package.go vendored
View File

@@ -3,6 +3,7 @@ package debian
import (
"bytes"
"fmt"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/smira/aptly/utils"
"github.com/ugorji/go/codec"
@@ -19,8 +20,8 @@ type PackageFile struct {
}
// Verify that package file is present and correct
func (f *PackageFile) Verify(packageRepo *Repository) (bool, error) {
poolPath, err := packageRepo.PoolPath(f.Filename, f.Checksums.MD5)
func (f *PackageFile) Verify(packagePool aptly.PackagePool) (bool, error) {
poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
if err != nil {
return false, err
}
@@ -360,19 +361,19 @@ func (p *Package) Equals(p2 *Package) bool {
}
// LinkFromPool links package file from pool to dist's pool location
func (p *Package) LinkFromPool(packageRepo *Repository, prefix string, component string) error {
func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool, prefix string, component string) error {
poolDir, err := p.PoolDirectory()
if err != nil {
return err
}
for i, f := range p.Files {
sourcePath, err := packageRepo.PoolPath(f.Filename, f.Checksums.MD5)
sourcePath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
if err != nil {
return err
}
relPath, err := packageRepo.LinkFromPool(prefix, component, sourcePath, poolDir)
relPath, err := publishedStorage.LinkFromPool(prefix, component, poolDir, packagePool, sourcePath)
if err != nil {
return err
}
@@ -418,16 +419,16 @@ type PackageDownloadTask struct {
// DownloadList returns list of missing package files for download in format
// [[srcpath, dstpath]]
func (p *Package) DownloadList(packageRepo *Repository) (result []PackageDownloadTask, err error) {
func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageDownloadTask, err error) {
result = make([]PackageDownloadTask, 0, 1)
for _, f := range p.Files {
poolPath, err := packageRepo.PoolPath(f.Filename, f.Checksums.MD5)
poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
if err != nil {
return nil, err
}
verified, err := f.Verify(packageRepo)
verified, err := f.Verify(packagePool)
if err != nil {
return nil, err
}
@@ -441,11 +442,11 @@ func (p *Package) DownloadList(packageRepo *Repository) (result []PackageDownloa
}
// VerifyFiles verifies that all package files have neen correctly downloaded
func (p *Package) VerifyFiles(packageRepo *Repository) (result bool, err error) {
func (p *Package) VerifyFiles(packagePool aptly.PackagePool) (result bool, err error) {
result = true
for _, f := range p.Files {
result, err = f.Verify(packageRepo)
result, err = f.Verify(packagePool)
if err != nil || !result {
return
}
@@ -455,12 +456,12 @@ func (p *Package) VerifyFiles(packageRepo *Repository) (result bool, err error)
}
// FilepathList returns list of paths to files in package repository
func (p *Package) FilepathList(packageRepo *Repository) ([]string, error) {
func (p *Package) FilepathList(packagePool aptly.PackagePool) ([]string, error) {
var err error
result := make([]string, len(p.Files))
for i, f := range p.Files {
result[i], err = packageRepo.RelativePoolPath(f.Filename, f.Checksums.MD5)
result[i], err = packagePool.RelativePath(f.Filename, f.Checksums.MD5)
if err != nil {
return nil, err
}

36
debian/package_test.go vendored
View File

@@ -3,6 +3,7 @@ package debian
import (
"bytes"
"github.com/smira/aptly/database"
"github.com/smira/aptly/files"
"github.com/smira/aptly/utils"
. "launchpad.net/gocheck"
"os"
@@ -24,11 +25,11 @@ func (s *PackageSuite) SetUpTest(c *C) {
}
func (s *PackageSuite) TestPackageFileVerify(c *C) {
packageRepo := NewRepository(c.MkDir())
packagePool := files.NewPackagePool(c.MkDir())
p := NewPackageFromControlFile(s.stanza)
poolPath, _ := packageRepo.PoolPath(p.Files[0].Filename, p.Files[0].Checksums.MD5)
poolPath, _ := packagePool.Path(p.Files[0].Filename, p.Files[0].Checksums.MD5)
result, err := p.Files[0].Verify(packageRepo)
result, err := p.Files[0].Verify(packagePool)
c.Check(err, IsNil)
c.Check(result, Equals, false)
@@ -40,20 +41,20 @@ func (s *PackageSuite) TestPackageFileVerify(c *C) {
file.WriteString("abcde")
file.Close()
result, err = p.Files[0].Verify(packageRepo)
result, err = p.Files[0].Verify(packagePool)
c.Check(err, IsNil)
c.Check(result, Equals, false)
result, err = p.VerifyFiles(packageRepo)
result, err = p.VerifyFiles(packagePool)
c.Check(err, IsNil)
c.Check(result, Equals, false)
p.Files[0].Checksums.Size = 5
result, err = p.Files[0].Verify(packageRepo)
result, err = p.Files[0].Verify(packagePool)
c.Check(err, IsNil)
c.Check(result, Equals, true)
result, err = p.VerifyFiles(packageRepo)
result, err = p.VerifyFiles(packagePool)
c.Check(err, IsNil)
c.Check(result, Equals, true)
}
@@ -226,10 +227,11 @@ func (s *PackageSuite) TestPoolDirectory(c *C) {
}
func (s *PackageSuite) TestLinkFromPool(c *C) {
packageRepo := NewRepository(c.MkDir())
packagePool := files.NewPackagePool(c.MkDir())
publishedStorage := files.NewPublishedStorage(c.MkDir())
p := NewPackageFromControlFile(s.stanza)
poolPath, _ := packageRepo.PoolPath(p.Files[0].Filename, p.Files[0].Checksums.MD5)
poolPath, _ := packagePool.Path(p.Files[0].Filename, p.Files[0].Checksums.MD5)
err := os.MkdirAll(filepath.Dir(poolPath), 0755)
c.Assert(err, IsNil)
@@ -237,32 +239,32 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
c.Assert(err, IsNil)
file.Close()
err = p.LinkFromPool(packageRepo, "", "non-free")
err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free")
c.Check(err, IsNil)
c.Check(p.Files[0].Filename, Equals, "pool/non-free/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
p.IsSource = true
err = p.LinkFromPool(packageRepo, "", "non-free")
err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free")
c.Check(err, IsNil)
c.Check(p.Extra["Directory"], Equals, "pool/non-free/a/alien-arena")
}
func (s *PackageSuite) TestFilepathList(c *C) {
packageRepo := NewRepository(c.MkDir())
packagePool := files.NewPackagePool(c.MkDir())
p := NewPackageFromControlFile(s.stanza)
list, err := p.FilepathList(packageRepo)
list, err := p.FilepathList(packagePool)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"1e/8c/alien-arena-common_7.40-2_i386.deb"})
}
func (s *PackageSuite) TestDownloadList(c *C) {
packageRepo := NewRepository(c.MkDir())
packagePool := files.NewPackagePool(c.MkDir())
p := NewPackageFromControlFile(s.stanza)
p.Files[0].Checksums.Size = 5
poolPath, _ := packageRepo.PoolPath(p.Files[0].Filename, p.Files[0].Checksums.MD5)
poolPath, _ := packagePool.Path(p.Files[0].Filename, p.Files[0].Checksums.MD5)
list, err := p.DownloadList(packageRepo)
list, err := p.DownloadList(packagePool)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []PackageDownloadTask{
PackageDownloadTask{
@@ -281,7 +283,7 @@ func (s *PackageSuite) TestDownloadList(c *C) {
file.WriteString("abcde")
file.Close()
list, err = p.DownloadList(packageRepo)
list, err = p.DownloadList(packagePool)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []PackageDownloadTask{})
}

35
debian/publish.go vendored
View File

@@ -5,6 +5,7 @@ import (
"bytes"
"code.google.com/p/go-uuid/uuid"
"fmt"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/smira/aptly/utils"
"github.com/ugorji/go/codec"
@@ -92,13 +93,13 @@ func (p *PublishedRepo) Decode(input []byte) error {
}
// Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageCollection, signer utils.Signer) error {
err := repo.MkDir(filepath.Join(p.Prefix, "pool"))
func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage aptly.PublishedStorage, packageCollection *PackageCollection, signer utils.Signer) error {
err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
if err != nil {
return err
}
basePath := filepath.Join(p.Prefix, "dists", p.Distribution)
err = repo.MkDir(basePath)
err = publishedStorage.MkDir(basePath)
if err != nil {
return err
}
@@ -133,12 +134,12 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl
} else {
relativePath = filepath.Join(p.Component, fmt.Sprintf("binary-%s", arch), "Packages")
}
err = repo.MkDir(filepath.Dir(filepath.Join(basePath, relativePath)))
err = publishedStorage.MkDir(filepath.Dir(filepath.Join(basePath, relativePath)))
if err != nil {
return err
}
packagesFile, err := repo.CreateFile(filepath.Join(basePath, relativePath))
packagesFile, err := publishedStorage.CreateFile(filepath.Join(basePath, relativePath))
if err != nil {
return fmt.Errorf("unable to creates Packages file: %s", err)
}
@@ -147,7 +148,7 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl
err = list.ForEach(func(pkg *Package) error {
if pkg.MatchesArchitecture(arch) {
err = pkg.LinkFromPool(repo, p.Prefix, p.Component)
err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, p.Component)
if err != nil {
return err
}
@@ -182,19 +183,19 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl
packagesFile.Close()
checksumInfo, err := repo.ChecksumsForFile(filepath.Join(basePath, relativePath))
checksumInfo, err := publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath))
if err != nil {
return fmt.Errorf("unable to collect checksums: %s", err)
}
generatedFiles[relativePath] = checksumInfo
checksumInfo, err = repo.ChecksumsForFile(filepath.Join(basePath, relativePath+".gz"))
checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+".gz"))
if err != nil {
return fmt.Errorf("unable to collect checksums: %s", err)
}
generatedFiles[relativePath+".gz"] = checksumInfo
checksumInfo, err = repo.ChecksumsForFile(filepath.Join(basePath, relativePath+".bz2"))
checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+".bz2"))
if err != nil {
return fmt.Errorf("unable to collect checksums: %s", err)
}
@@ -220,7 +221,7 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl
release["SHA256"] += fmt.Sprintf(" %s %8d %s\n", info.SHA256, info.Size, path)
}
releaseFile, err := repo.CreateFile(filepath.Join(basePath, "Release"))
releaseFile, err := publishedStorage.CreateFile(filepath.Join(basePath, "Release"))
if err != nil {
return fmt.Errorf("unable to create Release file: %s", err)
}
@@ -258,23 +259,23 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl
// RemoveFiles removes files that were created by Publish
//
// It can remove prefix fully, and part of pool (for specific component)
func (p *PublishedRepo) RemoveFiles(repo *Repository, removePrefix, removePoolComponent bool) error {
func (p *PublishedRepo) RemoveFiles(publishedStorage aptly.PublishedStorage, removePrefix, removePoolComponent bool) error {
if removePrefix {
err := repo.RemoveDirs(filepath.Join(p.Prefix, "dists"))
err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists"))
if err != nil {
return err
}
return repo.RemoveDirs(filepath.Join(p.Prefix, "pool"))
return publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool"))
}
err := repo.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution))
err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution))
if err != nil {
return err
}
if removePoolComponent {
err = repo.RemoveDirs(filepath.Join(p.Prefix, "pool", p.Component))
err = publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool", p.Component))
if err != nil {
return err
}
@@ -404,7 +405,7 @@ func (collection *PublishedRepoCollection) Len() int {
}
// Remove removes published repository, cleaning up directories, files
func (collection *PublishedRepoCollection) Remove(packageRepo *Repository, prefix, distribution string) error {
func (collection *PublishedRepoCollection) Remove(publishedStorage aptly.PublishedStorage, prefix, distribution string) error {
repo, err := collection.ByPrefixDistribution(prefix, distribution)
if err != nil {
return err
@@ -427,7 +428,7 @@ func (collection *PublishedRepoCollection) Remove(packageRepo *Repository, prefi
}
}
err = repo.RemoveFiles(packageRepo, removePrefix, removePoolComponent)
err = repo.RemoveFiles(publishedStorage, removePrefix, removePoolComponent)
if err != nil {
return err
}

156
debian/publish_test.go vendored
View File

@@ -2,7 +2,9 @@ package debian
import (
"errors"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/smira/aptly/files"
. "launchpad.net/gocheck"
"os"
"path/filepath"
@@ -44,7 +46,9 @@ func (n *NullSigner) ClearSign(source string, destination string) error {
type PublishedRepoSuite struct {
PackageListMixinSuite
repo *PublishedRepo
packageRepo *Repository
root string
publishedStorage aptly.PublishedStorage
packagePool aptly.PackagePool
snapshot *Snapshot
db database.Storage
packageCollection *PackageCollection
@@ -57,7 +61,9 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
s.db, _ = database.OpenDB(c.MkDir())
s.packageRepo = NewRepository(c.MkDir())
s.root = c.MkDir()
s.publishedStorage = files.NewPublishedStorage(s.root)
s.packagePool = files.NewPackagePool(s.root)
repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
repo.packageRefs = s.reflist
@@ -71,7 +77,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
s.packageCollection.Update(s.p2)
s.packageCollection.Update(s.p3)
poolPath, _ := s.packageRepo.PoolPath(s.p1.Files[0].Filename, s.p1.Files[0].Checksums.MD5)
poolPath, _ := s.packagePool.Path(s.p1.Files[0].Filename, s.p1.Files[0].Checksums.MD5)
err := os.MkdirAll(filepath.Dir(poolPath), 0755)
f, err := os.Create(poolPath)
c.Assert(err, IsNil)
@@ -148,12 +154,12 @@ func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
}
func (s *PublishedRepoSuite) TestPublish(c *C) {
err := s.repo.Publish(s.packageRepo, s.packageCollection, &NullSigner{})
err := s.repo.Publish(s.packagePool, s.publishedStorage, s.packageCollection, &NullSigner{})
c.Assert(err, IsNil)
c.Check(s.repo.Architectures, DeepEquals, []string{"i386"})
rf, err := os.Open(filepath.Join(s.packageRepo.RootPath, "public/ppa/dists/squeeze/Release"))
rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
c.Assert(err, IsNil)
cfr := NewControlFileReader(rf)
@@ -164,7 +170,7 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
c.Check(st["Components"], Equals, "main")
c.Check(st["Architectures"], Equals, "i386")
pf, err := os.Open(filepath.Join(s.packageRepo.RootPath, "public/ppa/dists/squeeze/main/binary-i386/Packages"))
pf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages"))
c.Assert(err, IsNil)
cfr = NewControlFileReader(pf)
@@ -180,15 +186,15 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
c.Assert(err, IsNil)
c.Assert(st, IsNil)
_, err = os.Stat(filepath.Join(s.packageRepo.RootPath, "public/ppa/pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb"))
_, err = os.Stat(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb"))
c.Assert(err, IsNil)
}
func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
err := s.repo.Publish(s.packageRepo, s.packageCollection, nil)
err := s.repo.Publish(s.packagePool, s.publishedStorage, s.packageCollection, nil)
c.Assert(err, IsNil)
c.Check(filepath.Join(s.packageRepo.RootPath, "public/ppa/dists/squeeze/Release"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists)
}
func (s *PublishedRepoSuite) TestString(c *C) {
@@ -335,7 +341,8 @@ type PublishedRepoRemoveSuite struct {
db database.Storage
snapshotCollection *SnapshotCollection
collection *PublishedRepoCollection
packageRepo *Repository
root string
publishedStorage aptly.PublishedStorage
snap1 *Snapshot
repo1, repo2, repo3, repo4 *PublishedRepo
}
@@ -362,14 +369,15 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
s.collection.Add(s.repo3)
s.collection.Add(s.repo4)
s.packageRepo = NewRepository(c.MkDir())
s.packageRepo.MkDir("ppa/dists/anaconda")
s.packageRepo.MkDir("ppa/dists/meduza")
s.packageRepo.MkDir("ppa/dists/osminog")
s.packageRepo.MkDir("ppa/pool/main")
s.packageRepo.MkDir("ppa/pool/contrib")
s.packageRepo.MkDir("dists/anaconda")
s.packageRepo.MkDir("pool/main")
s.root = c.MkDir()
s.publishedStorage = files.NewPublishedStorage(s.root)
s.publishedStorage.MkDir("ppa/dists/anaconda")
s.publishedStorage.MkDir("ppa/dists/meduza")
s.publishedStorage.MkDir("ppa/dists/osminog")
s.publishedStorage.MkDir("ppa/pool/main")
s.publishedStorage.MkDir("ppa/pool/contrib")
s.publishedStorage.MkDir("dists/anaconda")
s.publishedStorage.MkDir("pool/main")
}
func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
@@ -377,54 +385,54 @@ func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
}
func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
s.repo1.RemoveFiles(s.packageRepo, false, false)
s.repo1.RemoveFiles(s.publishedStorage, false, false)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/osminog"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/anaconda"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/main"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
}
func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
s.repo1.RemoveFiles(s.packageRepo, false, true)
s.repo1.RemoveFiles(s.publishedStorage, false, true)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/osminog"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/anaconda"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/main"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
}
func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
s.repo1.RemoveFiles(s.packageRepo, true, true)
s.repo1.RemoveFiles(s.publishedStorage, true, true)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/osminog"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/anaconda"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/main"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
}
func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
s.repo2.RemoveFiles(s.packageRepo, true, true)
s.repo2.RemoveFiles(s.publishedStorage, true, true)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/main"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), Not(PathExists))
}
func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
err := s.collection.Remove(s.packageRepo, "ppa", "anaconda")
err := s.collection.Remove(s.publishedStorage, "ppa", "anaconda")
c.Check(err, IsNil)
_, err = s.collection.ByPrefixDistribution("ppa", "anaconda")
@@ -434,31 +442,31 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
_, err = collection.ByPrefixDistribution("ppa", "anaconda")
c.Check(err, ErrorMatches, ".*not found")
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/osminog"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/anaconda"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/main"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
err = s.collection.Remove(s.packageRepo, "ppa", "anaconda")
err = s.collection.Remove(s.publishedStorage, "ppa", "anaconda")
c.Check(err, ErrorMatches, ".*not found")
err = s.collection.Remove(s.packageRepo, "ppa", "meduza")
err = s.collection.Remove(s.publishedStorage, "ppa", "meduza")
c.Check(err, IsNil)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/osminog"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/anaconda"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/main"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
}
func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
err := s.collection.Remove(s.packageRepo, ".", "anaconda")
err := s.collection.Remove(s.publishedStorage, ".", "anaconda")
c.Check(err, IsNil)
_, err = s.collection.ByPrefixDistribution(".", "anaconda")
@@ -468,11 +476,11 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
_, err = collection.ByPrefixDistribution(".", "anaconda")
c.Check(err, ErrorMatches, ".*not found")
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/osminog"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), PathExists)
c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/"), Not(PathExists))
c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/"), Not(PathExists))
}

5
debian/remote.go vendored
View File

@@ -5,6 +5,7 @@ import (
"bytes"
"code.google.com/p/go-uuid/uuid"
"fmt"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/smira/aptly/utils"
"github.com/ugorji/go/codec"
@@ -297,7 +298,7 @@ ok:
}
// Download downloads all repo files
func (repo *RemoteRepo) Download(d utils.Downloader, packageCollection *PackageCollection, packageRepo *Repository, ignoreMismatch bool) error {
func (repo *RemoteRepo) Download(d utils.Downloader, packageCollection *PackageCollection, packagePool aptly.PackagePool, ignoreMismatch bool) error {
list := NewPackageList()
d.GetProgress().Printf("Downloading & parsing package files...\n")
@@ -394,7 +395,7 @@ func (repo *RemoteRepo) Download(d utils.Downloader, packageCollection *PackageC
downloadSize := int64(0)
err = list.ForEach(func(p *Package) error {
list, err := p.DownloadList(packageRepo)
list, err := p.DownloadList(packagePool)
if err != nil {
return err
}

26
debian/remote_test.go vendored
View File

@@ -2,7 +2,9 @@ package debian
import (
"errors"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/smira/aptly/files"
"github.com/smira/aptly/utils"
"io"
"io/ioutil"
@@ -70,7 +72,7 @@ type RemoteRepoSuite struct {
downloader *utils.FakeDownloader
db database.Storage
packageCollection *PackageCollection
packageRepo *Repository
packagePool aptly.PackagePool
}
var _ = Suite(&RemoteRepoSuite{})
@@ -81,7 +83,7 @@ func (s *RemoteRepoSuite) SetUpTest(c *C) {
s.downloader = utils.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
s.db, _ = database.OpenDB(c.MkDir())
s.packageCollection = NewPackageCollection(s.db)
s.packageRepo = NewRepository(c.MkDir())
s.packagePool = files.NewPackagePool(c.MkDir())
s.SetUpPackages()
}
@@ -240,7 +242,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
err = s.repo.Download(s.downloader, s.packageCollection, s.packageRepo, false)
err = s.repo.Download(s.downloader, s.packageCollection, s.packagePool, false)
c.Assert(err, IsNil)
c.Assert(s.downloader.Empty(), Equals, true)
c.Assert(s.repo.packageRefs, NotNil)
@@ -248,7 +250,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
pkg, err := s.packageCollection.ByKey(s.repo.packageRefs.Refs[0])
c.Assert(err, IsNil)
result, err := pkg.VerifyFiles(s.packageRepo)
result, err := pkg.VerifyFiles(s.packagePool)
c.Check(result, Equals, true)
c.Check(err, IsNil)
@@ -273,7 +275,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz", "abcd")
s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz", "abcde")
err = s.repo.Download(s.downloader, s.packageCollection, s.packageRepo, false)
err = s.repo.Download(s.downloader, s.packageCollection, s.packagePool, false)
c.Assert(err, IsNil)
c.Assert(s.downloader.Empty(), Equals, true)
c.Assert(s.repo.packageRefs, NotNil)
@@ -281,7 +283,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
pkg, err := s.packageCollection.ByKey(s.repo.packageRefs.Refs[0])
c.Assert(err, IsNil)
result, err := pkg.VerifyFiles(s.packageRepo)
result, err := pkg.VerifyFiles(s.packagePool)
c.Check(result, Equals, true)
c.Check(err, IsNil)
@@ -290,7 +292,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
pkg, err = s.packageCollection.ByKey(s.repo.packageRefs.Refs[1])
c.Assert(err, IsNil)
result, err = pkg.VerifyFiles(s.packageRepo)
result, err = pkg.VerifyFiles(s.packagePool)
c.Check(result, Equals, true)
c.Check(err, IsNil)
@@ -308,7 +310,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
err := s.flat.Fetch(downloader, nil)
c.Assert(err, IsNil)
err = s.flat.Download(downloader, s.packageCollection, s.packageRepo, false)
err = s.flat.Download(downloader, s.packageCollection, s.packagePool, false)
c.Assert(err, IsNil)
c.Assert(downloader.Empty(), Equals, true)
c.Assert(s.flat.packageRefs, NotNil)
@@ -316,7 +318,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
pkg, err := s.packageCollection.ByKey(s.flat.packageRefs.Refs[0])
c.Assert(err, IsNil)
result, err := pkg.VerifyFiles(s.packageRepo)
result, err := pkg.VerifyFiles(s.packagePool)
c.Check(result, Equals, true)
c.Check(err, IsNil)
@@ -342,7 +344,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
err := s.flat.Fetch(downloader, nil)
c.Assert(err, IsNil)
err = s.flat.Download(downloader, s.packageCollection, s.packageRepo, false)
err = s.flat.Download(downloader, s.packageCollection, s.packagePool, false)
c.Assert(err, IsNil)
c.Assert(downloader.Empty(), Equals, true)
c.Assert(s.flat.packageRefs, NotNil)
@@ -350,7 +352,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
pkg, err := s.packageCollection.ByKey(s.flat.packageRefs.Refs[0])
c.Assert(err, IsNil)
result, err := pkg.VerifyFiles(s.packageRepo)
result, err := pkg.VerifyFiles(s.packagePool)
c.Check(result, Equals, true)
c.Check(err, IsNil)
@@ -359,7 +361,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
pkg, err = s.packageCollection.ByKey(s.flat.packageRefs.Refs[1])
c.Assert(err, IsNil)
result, err = pkg.VerifyFiles(s.packageRepo)
result, err = pkg.VerifyFiles(s.packagePool)
c.Check(result, Equals, true)
c.Check(err, IsNil)

166
debian/repository.go vendored
View File

@@ -1,166 +0,0 @@
package debian
import (
"fmt"
"github.com/smira/aptly/utils"
"io/ioutil"
"os"
"path/filepath"
)
// Repository directory structure:
// <root>
// \- pool
// \- ab
// \- ae
// \- package.deb
// \- public
// \- dists
// \- squeeze
// \- Release
// \- main
// \- binary-i386
// \- Packages.bz2
// references packages from pool
// \- pool
// contains symlinks to main pool
// Repository abstract file system with package pool and published package repos
type Repository struct {
RootPath string
}
// NewRepository creates new instance of repository which specified root
func NewRepository(root string) *Repository {
return &Repository{RootPath: root}
}
// RelativePoolPath returns path relative to pool's root
func (r *Repository) RelativePoolPath(filename string, hashMD5 string) (string, error) {
filename = filepath.Base(filename)
if filename == "." || filename == "/" {
return "", fmt.Errorf("filename %s is invalid", filename)
}
if len(hashMD5) < 4 {
return "", fmt.Errorf("unable to compute pool location for filename %v, MD5 is missing", filename)
}
return filepath.Join(hashMD5[0:2], hashMD5[2:4], filename), nil
}
// PoolPath returns full path to package file in pool given any name and hash of file contents
func (r *Repository) PoolPath(filename string, hashMD5 string) (string, error) {
relative, err := r.RelativePoolPath(filename, hashMD5)
if err != nil {
return "", err
}
return filepath.Join(r.RootPath, "pool", relative), nil
}
// PublicPath returns root of public part of repository
func (r *Repository) PublicPath() string {
return filepath.Join(r.RootPath, "public")
}
// MkDir creates directory recursively under public path
func (r *Repository) MkDir(path string) error {
return os.MkdirAll(filepath.Join(r.RootPath, "public", path), 0755)
}
// CreateFile creates file for writing under public path
func (r *Repository) CreateFile(path string) (*os.File, error) {
return os.Create(filepath.Join(r.RootPath, "public", path))
}
// RemoveDirs removes directory structure under public path
func (r *Repository) RemoveDirs(path string) error {
filepath := filepath.Join(r.RootPath, "public", path)
fmt.Printf("Removing %s...\n", filepath)
return os.RemoveAll(filepath)
}
// LinkFromPool links package file from pool to dist's pool location
func (r *Repository) LinkFromPool(prefix string, component string, sourcePath string, poolDirectory string) (string, error) {
baseName := filepath.Base(sourcePath)
relPath := filepath.Join("pool", component, poolDirectory, baseName)
poolPath := filepath.Join(r.RootPath, "public", prefix, "pool", component, poolDirectory)
err := os.MkdirAll(poolPath, 0755)
if err != nil {
return "", err
}
_, err = os.Stat(filepath.Join(poolPath, baseName))
if err == nil { // already exists, skip
return relPath, nil
}
err = os.Link(sourcePath, filepath.Join(poolPath, baseName))
return relPath, err
}
// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
func (r *Repository) ChecksumsForFile(path string) (utils.ChecksumInfo, error) {
return utils.ChecksumsForFile(filepath.Join(r.RootPath, "public", path))
}
// PoolFilepathList returns file paths of all the files in the pool
func (r *Repository) PoolFilepathList(progress *utils.Progress) ([]string, error) {
poolPath := filepath.Join(r.RootPath, "pool")
dirs, err := ioutil.ReadDir(poolPath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
if len(dirs) == 0 {
return nil, nil
}
if progress != nil {
progress.InitBar(int64(len(dirs)), false)
defer progress.ShutdownBar()
}
result := []string{}
for _, dir := range dirs {
err = filepath.Walk(filepath.Join(poolPath, dir.Name()), func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
result = append(result, path[len(poolPath)+1:])
}
return nil
})
if err != nil {
return nil, err
}
if progress != nil {
progress.AddBar(1)
}
}
return result, nil
}
// PoolRemove deletes file in package pool
func (r *Repository) PoolRemove(path string) (size int64, err error) {
path = filepath.Join(r.RootPath, "pool", path)
info, err := os.Stat(path)
if err != nil {
return 0, err
}
err = os.Remove(path)
return info.Size(), err
}

184
debian/repository_test.go vendored
View File

@@ -1,184 +0,0 @@
package debian
import (
"io/ioutil"
. "launchpad.net/gocheck"
"os"
"path/filepath"
"syscall"
)
type RepositorySuite struct {
repo *Repository
}
var _ = Suite(&RepositorySuite{})
func (s *RepositorySuite) SetUpTest(c *C) {
s.repo = NewRepository(c.MkDir())
}
func (s *RepositorySuite) TestRelativePoolPath(c *C) {
path, err := s.repo.RelativePoolPath("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, IsNil)
c.Assert(path, Equals, "91/b1/package.deb")
_, err = s.repo.RelativePoolPath("/", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, ErrorMatches, ".*is invalid")
_, err = s.repo.RelativePoolPath("", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, ErrorMatches, ".*is invalid")
_, err = s.repo.RelativePoolPath("a/b/package.deb", "9")
c.Assert(err, ErrorMatches, ".*MD5 is missing")
}
func (s *RepositorySuite) TestPoolPath(c *C) {
path, err := s.repo.PoolPath("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, IsNil)
c.Assert(path, Equals, filepath.Join(s.repo.RootPath, "pool", "91/b1/package.deb"))
_, err = s.repo.PoolPath("/", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, ErrorMatches, ".*is invalid")
}
func (s *RepositorySuite) TestPublicPath(c *C) {
c.Assert(s.repo.PublicPath(), Equals, filepath.Join(s.repo.RootPath, "public"))
}
func (s *RepositorySuite) TestMkDir(c *C) {
err := s.repo.MkDir("ppa/dists/squeeze/")
c.Assert(err, IsNil)
_, err = os.Stat(filepath.Join(s.repo.RootPath, "public/ppa/dists/squeeze/"))
c.Assert(err, IsNil)
}
func (s *RepositorySuite) TestCreateFile(c *C) {
err := s.repo.MkDir("ppa/dists/squeeze/")
c.Assert(err, IsNil)
file, err := s.repo.CreateFile("ppa/dists/squeeze/Release")
c.Assert(err, IsNil)
defer file.Close()
_, err = os.Stat(filepath.Join(s.repo.RootPath, "public/ppa/dists/squeeze/Release"))
c.Assert(err, IsNil)
}
func (s *RepositorySuite) TestRemoveDirs(c *C) {
err := s.repo.MkDir("ppa/dists/squeeze/")
c.Assert(err, IsNil)
file, err := s.repo.CreateFile("ppa/dists/squeeze/Release")
c.Assert(err, IsNil)
defer file.Close()
err = s.repo.RemoveDirs("ppa/dists/")
_, err = os.Stat(filepath.Join(s.repo.RootPath, "public/ppa/dists/squeeze/Release"))
c.Assert(err, NotNil)
c.Assert(os.IsNotExist(err), Equals, true)
}
func (s *RepositorySuite) TestLinkFromPool(c *C) {
tests := []struct {
prefix string
component string
sourcePath string
poolDirectory string
expectedFilename string
}{
{ // package name regular
prefix: "",
component: "main",
sourcePath: "pool/01/ae/mars-invaders_1.03.deb",
poolDirectory: "m/mars-invaders",
expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
},
{ // lib-like filename
prefix: "",
component: "main",
sourcePath: "pool/01/ae/libmars-invaders_1.03.deb",
poolDirectory: "libm/libmars-invaders",
expectedFilename: "pool/main/libm/libmars-invaders/libmars-invaders_1.03.deb",
},
{ // duplicate link, shouldn't panic
prefix: "",
component: "main",
sourcePath: "pool/01/ae/mars-invaders_1.03.deb",
poolDirectory: "m/mars-invaders",
expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
},
{ // prefix & component
prefix: "ppa",
component: "contrib",
sourcePath: "pool/01/ae/libmars-invaders_1.04.deb",
poolDirectory: "libm/libmars-invaders",
expectedFilename: "pool/contrib/libm/libmars-invaders/libmars-invaders_1.04.deb",
},
}
for _, t := range tests {
t.sourcePath = filepath.Join(s.repo.RootPath, t.sourcePath)
err := os.MkdirAll(filepath.Dir(t.sourcePath), 0755)
c.Assert(err, IsNil)
err = ioutil.WriteFile(t.sourcePath, []byte("Contents"), 0644)
c.Assert(err, IsNil)
path, err := s.repo.LinkFromPool(t.prefix, t.component, t.sourcePath, t.poolDirectory)
c.Assert(err, IsNil)
c.Assert(path, Equals, t.expectedFilename)
st, err := os.Stat(filepath.Join(s.repo.RootPath, "public", t.prefix, t.expectedFilename))
c.Assert(err, IsNil)
info := st.Sys().(*syscall.Stat_t)
c.Check(int(info.Nlink), Equals, 2)
}
}
func (s *RepositorySuite) TestPoolFilepathList(c *C) {
list, err := s.repo.PoolFilepathList(nil)
c.Check(err, IsNil)
c.Check(list, IsNil)
os.MkdirAll(filepath.Join(s.repo.RootPath, "pool", "bd", "0b"), 0755)
os.MkdirAll(filepath.Join(s.repo.RootPath, "pool", "bd", "0a"), 0755)
os.MkdirAll(filepath.Join(s.repo.RootPath, "pool", "ae", "0c"), 0755)
list, err = s.repo.PoolFilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{})
ioutil.WriteFile(filepath.Join(s.repo.RootPath, "pool", "ae", "0c", "1.deb"), nil, 0644)
ioutil.WriteFile(filepath.Join(s.repo.RootPath, "pool", "ae", "0c", "2.deb"), nil, 0644)
ioutil.WriteFile(filepath.Join(s.repo.RootPath, "pool", "bd", "0a", "3.deb"), nil, 0644)
ioutil.WriteFile(filepath.Join(s.repo.RootPath, "pool", "bd", "0b", "4.deb"), nil, 0644)
list, err = s.repo.PoolFilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"ae/0c/1.deb", "ae/0c/2.deb", "bd/0a/3.deb", "bd/0b/4.deb"})
}
func (s *RepositorySuite) TestPoolRemove(c *C) {
os.MkdirAll(filepath.Join(s.repo.RootPath, "pool", "bd", "0b"), 0755)
os.MkdirAll(filepath.Join(s.repo.RootPath, "pool", "bd", "0a"), 0755)
os.MkdirAll(filepath.Join(s.repo.RootPath, "pool", "ae", "0c"), 0755)
ioutil.WriteFile(filepath.Join(s.repo.RootPath, "pool", "ae", "0c", "1.deb"), []byte("1"), 0644)
ioutil.WriteFile(filepath.Join(s.repo.RootPath, "pool", "ae", "0c", "2.deb"), []byte("22"), 0644)
ioutil.WriteFile(filepath.Join(s.repo.RootPath, "pool", "bd", "0a", "3.deb"), []byte("333"), 0644)
ioutil.WriteFile(filepath.Join(s.repo.RootPath, "pool", "bd", "0b", "4.deb"), []byte("4444"), 0644)
size, err := s.repo.PoolRemove("ae/0c/2.deb")
c.Check(err, IsNil)
c.Check(size, Equals, int64(2))
_, err = s.repo.PoolRemove("ae/0c/2.deb")
c.Check(err, ErrorMatches, ".*no such file or directory")
list, err := s.repo.PoolFilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"ae/0c/1.deb", "bd/0a/3.deb", "bd/0b/4.deb"})
}

19
files/files.go Normal file
View File

@@ -0,0 +1,19 @@
// Package files handles operation on filesystem for both public pool and published files
package files
// Repository directory structure:
// <root>
// \- pool
// \- ab
// \- ae
// \- package.deb
// \- public
// \- dists
// \- squeeze
// \- Release
// \- main
// \- binary-i386
// \- Packages.bz2
// references packages from pool
// \- pool
// contains symlinks to main pool

11
files/files_test.go Normal file
View File

@@ -0,0 +1,11 @@
package files
import (
. "launchpad.net/gocheck"
"testing"
)
// Launch gocheck tests
func Test(t *testing.T) {
TestingT(t)
}

105
files/package_pool.go Normal file
View File

@@ -0,0 +1,105 @@
package files
import (
"fmt"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/utils"
"io/ioutil"
"os"
"path/filepath"
)
// PackagePool is deduplicated storage of package files on filesystem
type PackagePool struct {
rootPath string
}
// Check interface
var (
_ aptly.PackagePool = (*PackagePool)(nil)
)
// NewPackagePool creates new instance of PackagePool which specified root
func NewPackagePool(root string) *PackagePool {
return &PackagePool{rootPath: filepath.Join(root, "pool")}
}
// RelativePath returns path relative to pool's root for package files given MD5 and original filename
func (pool *PackagePool) RelativePath(filename string, hashMD5 string) (string, error) {
filename = filepath.Base(filename)
if filename == "." || filename == "/" {
return "", fmt.Errorf("filename %s is invalid", filename)
}
if len(hashMD5) < 4 {
return "", fmt.Errorf("unable to compute pool location for filename %v, MD5 is missing", filename)
}
return filepath.Join(hashMD5[0:2], hashMD5[2:4], filename), nil
}
// Path returns full path to package file in pool given any name and hash of file contents
func (p *PackagePool) Path(filename string, hashMD5 string) (string, error) {
relative, err := p.RelativePath(filename, hashMD5)
if err != nil {
return "", err
}
return filepath.Join(p.rootPath, relative), nil
}
// FilepathList returns file paths of all the files in the pool
func (p *PackagePool) FilepathList(progress *utils.Progress) ([]string, error) {
dirs, err := ioutil.ReadDir(p.rootPath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
if len(dirs) == 0 {
return nil, nil
}
if progress != nil {
progress.InitBar(int64(len(dirs)), false)
defer progress.ShutdownBar()
}
result := []string{}
for _, dir := range dirs {
err = filepath.Walk(filepath.Join(p.rootPath, dir.Name()), func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
result = append(result, path[len(p.rootPath)+1:])
}
return nil
})
if err != nil {
return nil, err
}
if progress != nil {
progress.AddBar(1)
}
}
return result, nil
}
// Remove deletes file in package pool returns its size
func (p *PackagePool) Remove(path string) (size int64, err error) {
path = filepath.Join(p.rootPath, path)
info, err := os.Stat(path)
if err != nil {
return 0, err
}
err = os.Remove(path)
return info.Size(), err
}

86
files/package_pool_test.go Normal file
View File

@@ -0,0 +1,86 @@
package files
import (
"io/ioutil"
. "launchpad.net/gocheck"
"os"
"path/filepath"
)
type PackagePoolSuite struct {
pool *PackagePool
}
var _ = Suite(&PackagePoolSuite{})
func (s *PackagePoolSuite) SetUpTest(c *C) {
s.pool = NewPackagePool(c.MkDir())
}
func (s *PackagePoolSuite) TestRelativePath(c *C) {
path, err := s.pool.RelativePath("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, IsNil)
c.Assert(path, Equals, "91/b1/package.deb")
_, err = s.pool.RelativePath("/", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, ErrorMatches, ".*is invalid")
_, err = s.pool.RelativePath("", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, ErrorMatches, ".*is invalid")
_, err = s.pool.RelativePath("a/b/package.deb", "9")
c.Assert(err, ErrorMatches, ".*MD5 is missing")
}
func (s *PackagePoolSuite) TestPath(c *C) {
path, err := s.pool.Path("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, IsNil)
c.Assert(path, Equals, filepath.Join(s.pool.rootPath, "91/b1/package.deb"))
_, err = s.pool.Path("/", "91b1a1480b90b9e269ca44d897b12575")
c.Assert(err, ErrorMatches, ".*is invalid")
}
func (s *PackagePoolSuite) TestFilepathList(c *C) {
list, err := s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, IsNil)
os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0b"), 0755)
os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0a"), 0755)
os.MkdirAll(filepath.Join(s.pool.rootPath, "ae", "0c"), 0755)
list, err = s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{})
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "1.deb"), nil, 0644)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "2.deb"), nil, 0644)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0a", "3.deb"), nil, 0644)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0b", "4.deb"), nil, 0644)
list, err = s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"ae/0c/1.deb", "ae/0c/2.deb", "bd/0a/3.deb", "bd/0b/4.deb"})
}
func (s *PackagePoolSuite) TestRemove(c *C) {
os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0b"), 0755)
os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0a"), 0755)
os.MkdirAll(filepath.Join(s.pool.rootPath, "ae", "0c"), 0755)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "1.deb"), []byte("1"), 0644)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "2.deb"), []byte("22"), 0644)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0a", "3.deb"), []byte("333"), 0644)
ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0b", "4.deb"), []byte("4444"), 0644)
size, err := s.pool.Remove("ae/0c/2.deb")
c.Check(err, IsNil)
c.Check(size, Equals, int64(2))
_, err = s.pool.Remove("ae/0c/2.deb")
c.Check(err, ErrorMatches, ".*no such file or directory")
list, err := s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"ae/0c/1.deb", "bd/0a/3.deb", "bd/0b/4.deb"})
}

83
files/public.go Normal file
View File

@@ -0,0 +1,83 @@
package files
import (
"fmt"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/utils"
"os"
"path/filepath"
)
// PublishedStorage abstract file system with public dirs (published repos)
type PublishedStorage struct {
rootPath string
}
// Check interface
var (
_ aptly.PublishedStorage = (*PublishedStorage)(nil)
)
// NewPublishedStorage creates new instance of PublishedStorage which specified root
func NewPublishedStorage(root string) *PublishedStorage {
return &PublishedStorage{rootPath: filepath.Join(root, "public")}
}
// PublicPath returns root of public part
func (storage *PublishedStorage) PublicPath() string {
return storage.rootPath
}
// MkDir creates directory recursively under public path
func (storage *PublishedStorage) MkDir(path string) error {
return os.MkdirAll(filepath.Join(storage.rootPath, path), 0755)
}
// CreateFile creates file for writing under public path
func (storage *PublishedStorage) CreateFile(path string) (*os.File, error) {
return os.Create(filepath.Join(storage.rootPath, path))
}
// RemoveDirs removes directory structure under public path
func (storage *PublishedStorage) RemoveDirs(path string) error {
filepath := filepath.Join(storage.rootPath, path)
fmt.Printf("Removing %s...\n", filepath)
return os.RemoveAll(filepath)
}
// LinkFromPool links package file from pool to dist's pool location
//
// prefix is publishing prefix for this repo (e.g. empty or "ppa/")
// component is component name when publishing (e.g. main)
// poolDirectory is desired location in pool (like liba/libav/)
// sourcePool is instance of aptly.PackagePool
// sourcePath is filepath to package file in package pool
//
// LinkFromPool returns relative path for the published file to be included in package index
func (storage *PublishedStorage) LinkFromPool(prefix string, component string, poolDirectory string, sourcePool aptly.PackagePool, sourcePath string) (string, error) {
// verify that package pool is local pool is filesystem pool
_ = sourcePool.(*PackagePool)
baseName := filepath.Base(sourcePath)
relPath := filepath.Join("pool", component, poolDirectory, baseName)
poolPath := filepath.Join(storage.rootPath, prefix, "pool", component, poolDirectory)
err := os.MkdirAll(poolPath, 0755)
if err != nil {
return "", err
}
_, err = os.Stat(filepath.Join(poolPath, baseName))
if err == nil { // already exists, skip
return relPath, nil
}
err = os.Link(sourcePath, filepath.Join(poolPath, baseName))
return relPath, err
}
// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
func (storage *PublishedStorage) ChecksumsForFile(path string) (utils.ChecksumInfo, error) {
return utils.ChecksumsForFile(filepath.Join(storage.rootPath, path))
}

121
files/public_test.go Normal file
View File

@@ -0,0 +1,121 @@
package files
import (
"io/ioutil"
. "launchpad.net/gocheck"
"os"
"path/filepath"
"syscall"
)
type PublishedStorageSuite struct {
root string
storage *PublishedStorage
}
var _ = Suite(&PublishedStorageSuite{})
func (s *PublishedStorageSuite) SetUpTest(c *C) {
s.root = c.MkDir()
s.storage = NewPublishedStorage(s.root)
}
func (s *PublishedStorageSuite) TestPublicPath(c *C) {
c.Assert(s.storage.PublicPath(), Equals, filepath.Join(s.root, "public"))
}
func (s *PublishedStorageSuite) TestMkDir(c *C) {
err := s.storage.MkDir("ppa/dists/squeeze/")
c.Assert(err, IsNil)
_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/"))
c.Assert(err, IsNil)
}
func (s *PublishedStorageSuite) TestCreateFile(c *C) {
err := s.storage.MkDir("ppa/dists/squeeze/")
c.Assert(err, IsNil)
file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
c.Assert(err, IsNil)
defer file.Close()
_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
c.Assert(err, IsNil)
}
func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
err := s.storage.MkDir("ppa/dists/squeeze/")
c.Assert(err, IsNil)
file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
c.Assert(err, IsNil)
defer file.Close()
err = s.storage.RemoveDirs("ppa/dists/")
_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
c.Assert(err, NotNil)
c.Assert(os.IsNotExist(err), Equals, true)
}
func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
tests := []struct {
prefix string
component string
sourcePath string
poolDirectory string
expectedFilename string
}{
{ // package name regular
prefix: "",
component: "main",
sourcePath: "pool/01/ae/mars-invaders_1.03.deb",
poolDirectory: "m/mars-invaders",
expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
},
{ // lib-like filename
prefix: "",
component: "main",
sourcePath: "pool/01/ae/libmars-invaders_1.03.deb",
poolDirectory: "libm/libmars-invaders",
expectedFilename: "pool/main/libm/libmars-invaders/libmars-invaders_1.03.deb",
},
{ // duplicate link, shouldn't panic
prefix: "",
component: "main",
sourcePath: "pool/01/ae/mars-invaders_1.03.deb",
poolDirectory: "m/mars-invaders",
expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
},
{ // prefix & component
prefix: "ppa",
component: "contrib",
sourcePath: "pool/01/ae/libmars-invaders_1.04.deb",
poolDirectory: "libm/libmars-invaders",
expectedFilename: "pool/contrib/libm/libmars-invaders/libmars-invaders_1.04.deb",
},
}
pool := NewPackagePool(s.root)
for _, t := range tests {
t.sourcePath = filepath.Join(s.root, t.sourcePath)
err := os.MkdirAll(filepath.Dir(t.sourcePath), 0755)
c.Assert(err, IsNil)
err = ioutil.WriteFile(t.sourcePath, []byte("Contents"), 0644)
c.Assert(err, IsNil)
path, err := s.storage.LinkFromPool(t.prefix, t.component, t.poolDirectory, pool, t.sourcePath)
c.Assert(err, IsNil)
c.Assert(path, Equals, t.expectedFilename)
st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename))
c.Assert(err, IsNil)
info := st.Sys().(*syscall.Stat_t)
c.Check(int(info.Nlink), Equals, 2)
}
}

8
main.go
View File

@@ -4,8 +4,10 @@ import (
"fmt"
"github.com/gonuts/commander"
"github.com/gonuts/flag"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/smira/aptly/debian"
"github.com/smira/aptly/files"
"github.com/smira/aptly/utils"
"os"
"path/filepath"
@@ -48,7 +50,8 @@ take snapshots and publish them back as Debian repositories.`,
var context struct {
downloader utils.Downloader
database database.Storage
packageRepository *debian.Repository
packagePool aptly.PackagePool
publishedStorage aptly.PublishedStorage
dependencyOptions int
architecturesList []string
}
@@ -122,7 +125,8 @@ func main() {
}
defer context.database.Close()
context.packageRepository = debian.NewRepository(utils.Config.RootDir)
context.packagePool = files.NewPackagePool(utils.Config.RootDir)
context.publishedStorage = files.NewPublishedStorage(utils.Config.RootDir)
err = cmd.Dispatch(cmd.Flag.Args())
if err != nil {