New upstream version 1.1.1

deb/changes.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -9,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -23,7 +25,7 @@ type Changes struct {
 	Binary                []string
 	Architectures         []string
 	Stanza                Stanza
-	SignatureKeys         []utils.GpgKey
+	SignatureKeys         []pgp.Key
 }
 
 // NewChanges moves .changes file into temporary directory and creates Changes structure
@@ -50,7 +52,7 @@ func NewChanges(path string) (*Changes, error) {
 }
 
 // VerifyAndParse does optional signature verification and parses changes files
-func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier utils.Verifier) error {
+func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier pgp.Verifier) error {
 	input, err := os.Open(filepath.Join(c.TempDir, c.ChangesName))
 	if err != nil {
 		return err
@@ -69,7 +71,8 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 	}
 
 	if isClearSigned && !ignoreSignature {
-		keyInfo, err := verifier.VerifyClearsigned(input, false)
+		var keyInfo *pgp.KeyInfo
+		keyInfo, err = verifier.VerifyClearsigned(input, false)
 		if err != nil {
 			return err
 		}
@@ -78,7 +81,7 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 		c.SignatureKeys = keyInfo.GoodKeys
 	}
 
-	var text *os.File
+	var text io.ReadCloser
 
 	if isClearSigned {
 		text, err = verifier.ExtractClearsigned(input)
@@ -103,11 +106,7 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 	c.Architectures = strings.Fields(c.Stanza["Architecture"])
 
 	c.Files, err = c.Files.ParseSumFields(c.Stanza)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return err
 }
 
 // Prepare creates temporary directory, copies file there and verifies checksums
@@ -173,7 +172,7 @@ func (c *Changes) PackageQuery() (PackageQuery, error) {
 
 	// if c.Source is empty, this would never match
 	sourceQuery := &AndQuery{
-		L: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"},
+		L: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: ArchitectureSource},
 		R: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Source},
 	}
 
@@ -181,8 +180,7 @@ func (c *Changes) PackageQuery() (PackageQuery, error) {
 	if len(c.Binary) > 0 {
 		binaryQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Binary[0]}
 		// matching debug ddeb packages, they're not present in the Binary field
-		var ddebQuery PackageQuery
-		ddebQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: fmt.Sprintf("%s-dbgsym", c.Binary[0])}
+		var ddebQuery PackageQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: fmt.Sprintf("%s-dbgsym", c.Binary[0])}
 
 		for _, binary := range c.Binary[1:] {
 			binaryQuery = &OrQuery{
@@ -206,7 +204,7 @@ func (c *Changes) PackageQuery() (PackageQuery, error) {
 		}
 
 		binaryQuery = &AndQuery{
-			L: &NotQuery{Q: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"}},
+			L: &NotQuery{Q: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: ArchitectureSource}},
 			R: binaryQuery}
 	}
 

deb/checksum_collection.go

@@ -0,0 +1,67 @@
+package deb
+
+import (
+	"bytes"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/utils"
+	"github.com/ugorji/go/codec"
+)
+
+// ChecksumCollection does management of ChecksumInfo in DB
+type ChecksumCollection struct {
+	db          database.Storage
+	codecHandle *codec.MsgpackHandle
+}
+
+// NewChecksumCollection creates new ChecksumCollection and binds it to database
+func NewChecksumCollection(db database.Storage) *ChecksumCollection {
+	return &ChecksumCollection{
+		db:          db,
+		codecHandle: &codec.MsgpackHandle{},
+	}
+}
+
+func (collection *ChecksumCollection) dbKey(path string) []byte {
+	return []byte("C" + path)
+}
+
+// Get finds checksums in DB by path
+func (collection *ChecksumCollection) Get(path string) (*utils.ChecksumInfo, error) {
+	encoded, err := collection.db.Get(collection.dbKey(path))
+	if err != nil {
+		if err == database.ErrNotFound {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	c := &utils.ChecksumInfo{}
+
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
+	err = decoder.Decode(c)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// Update adds or updates information about checksum in DB
+func (collection *ChecksumCollection) Update(path string, c *utils.ChecksumInfo) error {
+	var encodeBuffer bytes.Buffer
+
+	encoder := codec.NewEncoder(&encodeBuffer, collection.codecHandle)
+	err := encoder.Encode(c)
+	if err != nil {
+		return err
+	}
+
+	return collection.db.Put(collection.dbKey(path), encodeBuffer.Bytes())
+}
+
+// Check interface
+var (
+	_ aptly.ChecksumStorage = &ChecksumCollection{}
+)

deb/checksum_collection_test.go

@@ -0,0 +1,47 @@
+package deb
+
+import (
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type ChecksumCollectionSuite struct {
+	collection *ChecksumCollection
+	c          utils.ChecksumInfo
+	db         database.Storage
+}
+
+var _ = Suite(&ChecksumCollectionSuite{})
+
+func (s *ChecksumCollectionSuite) SetUpTest(c *C) {
+	s.c = utils.ChecksumInfo{
+		Size:   124,
+		MD5:    "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+		SHA1:   "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+		SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+	}
+	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.collection = NewChecksumCollection(s.db)
+}
+
+func (s *ChecksumCollectionSuite) TearDownTest(c *C) {
+	s.db.Close()
+}
+
+func (s *ChecksumCollectionSuite) TestFlow(c *C) {
+	// checksum not stored
+	checksum, err := s.collection.Get("some/path")
+	c.Assert(err, IsNil)
+	c.Check(checksum, IsNil)
+
+	// store checksum
+	err = s.collection.Update("some/path", &s.c)
+	c.Assert(err, IsNil)
+
+	// load it back
+	checksum, err = s.collection.Get("some/path")
+	c.Assert(err, IsNil)
+	c.Check(*checksum, DeepEquals, s.c)
+}

deb/collections.go

@@ -15,6 +15,7 @@ type CollectionFactory struct {
 	snapshots      *SnapshotCollection
 	localRepos     *LocalRepoCollection
 	publishedRepos *PublishedRepoCollection
+	checksums      *ChecksumCollection
 }
 
 // NewCollectionFactory creates new factory
@@ -89,6 +90,18 @@ func (factory *CollectionFactory) PublishedRepoCollection() *PublishedRepoCollec
 	return factory.publishedRepos
 }
 
+// ChecksumCollection returns (or creates) new ChecksumCollection
+func (factory *CollectionFactory) ChecksumCollection() *ChecksumCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
+	if factory.checksums == nil {
+		factory.checksums = NewChecksumCollection(factory.db)
+	}
+
+	return factory.checksums
+}
+
 // Flush removes all references to collections, so that memory could be reclaimed
 func (factory *CollectionFactory) Flush() {
 	factory.Lock()
@@ -99,4 +112,5 @@ func (factory *CollectionFactory) Flush() {
 	factory.remoteRepos = nil
 	factory.publishedRepos = nil
 	factory.packages = nil
+	factory.checksums = nil
 }

deb/contents.go

@@ -26,8 +26,8 @@ func NewContentsIndex(db database.Storage) *ContentsIndex {
 }
 
 // Push adds package to contents index, calculating package contents as required
-func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool) error {
-	contents := p.Contents(packagePool)
+func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool, progress aptly.Progress) error {
+	contents := p.Contents(packagePool, progress)
 	qualifiedName := []byte(p.QualifiedName())
 
 	for _, path := range contents {

deb/deb.go

@@ -12,12 +12,20 @@ import (
 
 	"github.com/h2non/filetype/matchers"
 	"github.com/mkrautz/goar"
+	"github.com/pkg/errors"
 
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/go-xz"
 	"github.com/smira/lzma"
 )
 
+// Source kinds
+const (
+	SourceSnapshot   = "snapshot"
+	SourceLocalRepo  = "local"
+	SourceRemoteRepo = "repo"
+)
+
 // GetControlFileFromDeb reads control file from deb package
 func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	file, err := os.Open(packageFile)
@@ -68,7 +76,7 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 }
 
 // GetControlFileFromDsc reads control file from dsc package
-func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, error) {
+func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error) {
 	file, err := os.Open(dscFile)
 	if err != nil {
 		return nil, err
@@ -82,7 +90,7 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 		return nil, err
 	}
 
-	var text *os.File
+	var text io.ReadCloser
 
 	if isClearSigned {
 		text, err = verifier.ExtractClearsigned(file)
@@ -105,13 +113,7 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 }
 
 // GetContentsFromDeb returns list of files installed by .deb package
-func GetContentsFromDeb(packageFile string) ([]string, error) {
-	file, err := os.Open(packageFile)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
+func GetContentsFromDeb(file io.Reader, packageFile string) ([]string, error) {
 	library := ar.NewReader(file)
 	for {
 		header, err := library.Next()
@@ -119,7 +121,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
 			return nil, fmt.Errorf("unable to find data.tar.* part in %s", packageFile)
 		}
 		if err != nil {
-			return nil, fmt.Errorf("unable to read .deb archive from %s: %s", packageFile, err)
+			return nil, errors.Wrapf(err, "unable to read .deb archive from %s", packageFile)
 		}
 
 		if strings.HasPrefix(header.Name, "data.tar") {
@@ -142,7 +144,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
 				} else {
 					ungzip, err := gzip.NewReader(bufReader)
 					if err != nil {
-						return nil, fmt.Errorf("unable to ungzip data.tar.gz from %s: %s", packageFile, err)
+						return nil, errors.Wrapf(err, "unable to ungzip data.tar.gz from %s", packageFile)
 					}
 					defer ungzip.Close()
 					tarInput = ungzip
@@ -152,7 +154,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
 			case "data.tar.xz":
 				unxz, err := xz.NewReader(bufReader)
 				if err != nil {
-					return nil, fmt.Errorf("unable to unxz data.tar.xz from %s: %s", packageFile, err)
+					return nil, errors.Wrapf(err, "unable to unxz data.tar.xz from %s", packageFile)
 				}
 				defer unxz.Close()
 				tarInput = unxz
@@ -172,16 +174,14 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
 					return results, nil
 				}
 				if err != nil {
-					return nil, fmt.Errorf("unable to read .tar archive from %s: %s", packageFile, err)
+					return nil, errors.Wrapf(err, "unable to read .tar archive from %s", packageFile)
 				}
 
 				if tarHeader.Typeflag == tar.TypeDir {
 					continue
 				}
 
-				if strings.HasPrefix(tarHeader.Name, "./") {
-					tarHeader.Name = tarHeader.Name[2:]
-				}
+				tarHeader.Name = strings.TrimPrefix(tarHeader.Name[2:], "./")
 				results = append(results, tarHeader.Name)
 			}
 		}

deb/deb_test.go

@@ -1,10 +1,11 @@
 package deb
 
 import (
+	"os"
 	"path/filepath"
 	"runtime"
 
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/pgp"
 
 	. "gopkg.in/check.v1"
 )
@@ -38,7 +39,7 @@ func (s *DebSuite) TestGetControlFileFromDeb(c *C) {
 }
 
 func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
-	verifier := &utils.GpgVerifier{}
+	verifier := &pgp.GoVerifier{}
 
 	_, err := GetControlFileFromDsc("/no/such/file", verifier)
 	c.Check(err, ErrorMatches, ".*no such file or directory")
@@ -59,13 +60,19 @@ func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
 }
 
 func (s *DebSuite) TestGetContentsFromDeb(c *C) {
-	contents, err := GetContentsFromDeb(s.debFile)
+	f, err := os.Open(s.debFile)
+	c.Assert(err, IsNil)
+	contents, err := GetContentsFromDeb(f, s.debFile)
 	c.Check(err, IsNil)
 	c.Check(contents, DeepEquals, []string{"usr/share/doc/libboost-program-options-dev/changelog.gz",
 		"usr/share/doc/libboost-program-options-dev/copyright"})
+	c.Assert(f.Close(), IsNil)
 
-	contents, err = GetContentsFromDeb(s.debFile2)
+	f, err = os.Open(s.debFile2)
+	c.Assert(err, IsNil)
+	contents, err = GetContentsFromDeb(f, s.debFile2)
 	c.Check(err, IsNil)
 	c.Check(contents, DeepEquals, []string{"usr/bin/hardlink", "usr/share/man/man1/hardlink.1.gz",
 		"usr/share/doc/hardlink/changelog.gz", "usr/share/doc/hardlink/copyright", "usr/share/doc/hardlink/NEWS.Debian.gz"})
+	c.Assert(f.Close(), IsNil)
 }

deb/format.go

@@ -22,6 +22,8 @@ var (
 		"Version",
 		"Codename",
 		"Date",
+		"NotAutomatic",
+		"ButAutomaticUpgrades",
 		"Architectures",
 		"Architecture",
 		"Components",

deb/graph.go

@@ -33,9 +33,9 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	existingNodes := map[string]bool{}
 
 	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *RemoteRepo) error {
-		err := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
 		}
 
 		graph.AddNode("aptly", repo.UUID, map[string]string{
@@ -55,9 +55,9 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	}
 
 	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *LocalRepo) error {
-		err := collectionFactory.LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
 		}
 
 		graph.AddNode("aptly", repo.UUID, map[string]string{
@@ -81,13 +81,13 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	})
 
 	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
-		err := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
-			return err
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if e != nil {
+			return e
 		}
 
 		description := snapshot.Description
-		if snapshot.SourceKind == "repo" {
+		if snapshot.SourceKind == SourceRemoteRepo {
 			description = "Snapshot from repo"
 		}
 
@@ -99,7 +99,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 				snapshot.Name, description, snapshot.NumPackages(), labelEnd),
 		})
 
-		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
+		if snapshot.SourceKind == SourceRemoteRepo || snapshot.SourceKind == SourceLocalRepo || snapshot.SourceKind == SourceSnapshot {
 			for _, uuid := range snapshot.SourceIDs {
 				_, exists := existingNodes[uuid]
 				if exists {

deb/import.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -59,8 +60,9 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 }
 
 // ImportPackageFiles imports files into local repository
-func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier utils.Verifier,
-	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter, restriction PackageQuery) (processedFiles []string, failedFiles []string, err error) {
+func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier pgp.Verifier,
+	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter, restriction PackageQuery,
+	checksumStorage aptly.ChecksumStorage) (processedFiles []string, failedFiles []string, err error) {
 	if forceReplace {
 		list.PrepareIndex()
 	}
@@ -116,19 +118,24 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
+		var files PackageFiles
+
+		if isSourcePackage {
+			files = p.Files()
+		}
+
 		var checksums utils.ChecksumInfo
 		checksums, err = utils.ChecksumsForFile(file)
 		if err != nil {
 			return nil, nil, err
 		}
 
-		if isSourcePackage {
-			p.UpdateFiles(append(p.Files(), PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
-		} else {
-			p.UpdateFiles([]PackageFile{{Filename: filepath.Base(file), Checksums: checksums}})
+		mainPackageFile := PackageFile{
+			Filename:  filepath.Base(file),
+			Checksums: checksums,
 		}
 
-		err = pool.Import(file, checksums.MD5)
+		mainPackageFile.PoolPath, err = pool.Import(file, mainPackageFile.Filename, &mainPackageFile.Checksums, false, checksumStorage)
 		if err != nil {
 			reporter.Warning("Unable to import file %s into pool: %s", file, err)
 			failedFiles = append(failedFiles, file)
@@ -137,13 +144,10 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 
 		candidateProcessedFiles = append(candidateProcessedFiles, file)
 
-		// go over all files, except for the last one (.dsc/.deb itself)
-		for _, f := range p.Files() {
-			if filepath.Base(f.Filename) == filepath.Base(file) {
-				continue
-			}
-			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
-			err = pool.Import(sourceFile, f.Checksums.MD5)
+		// go over all the other files
+		for i := range files {
+			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(files[i].Filename))
+			files[i].PoolPath, err = pool.Import(sourceFile, files[i].Filename, &files[i].Checksums, false, checksumStorage)
 			if err != nil {
 				reporter.Warning("Unable to import file %s into pool: %s", sourceFile, err)
 				failedFiles = append(failedFiles, file)
@@ -157,6 +161,8 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
+		p.UpdateFiles(append(files, mainPackageFile))
+
 		if restriction != nil && !restriction.Matches(p) {
 			reporter.Warning("%s has been ignored as it doesn't match restriction", p)
 			failedFiles = append(failedFiles, file)

deb/index_files.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -48,7 +49,7 @@ func (file *indexFile) BufWriter() (*bufio.Writer, error) {
 	return file.w, nil
 }
 
-func (file *indexFile) Finalize(signer utils.Signer) error {
+func (file *indexFile) Finalize(signer pgp.Signer) error {
 	if file.w == nil {
 		if file.discardable {
 			return nil
@@ -155,7 +156,7 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 }
 
 func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
-	if arch == "source" {
+	if arch == ArchitectureSource {
 		udeb = false
 	}
 	key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb)
@@ -163,7 +164,7 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 	if !ok {
 		var relativePath string
 
-		if arch == "source" {
+		if arch == ArchitectureSource {
 			relativePath = filepath.Join(component, "source", "Sources")
 		} else {
 			if udeb {
@@ -188,7 +189,7 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 }
 
 func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexFile {
-	if arch == "source" {
+	if arch == ArchitectureSource {
 		udeb = false
 	}
 	key := fmt.Sprintf("ri-%s-%s-%v", component, arch, udeb)
@@ -196,7 +197,7 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 	if !ok {
 		var relativePath string
 
-		if arch == "source" {
+		if arch == ArchitectureSource {
 			relativePath = filepath.Join(component, "source", "Release")
 		} else {
 			if udeb {
@@ -221,7 +222,7 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 }
 
 func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *indexFile {
-	if arch == "source" {
+	if arch == ArchitectureSource {
 		udeb = false
 	}
 	key := fmt.Sprintf("ci-%s-%s-%v", component, arch, udeb)

deb/list.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"fmt"
 	"sort"
+	"strings"
 
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
@@ -20,6 +21,8 @@ const (
 	DepFollowAllVariants
 	// DepFollowBuild pulls build dependencies
 	DepFollowBuild
+	// DepVerboseResolve emits additional logs while dependencies are being resolved
+	DepVerboseResolve
 )
 
 // PackageList is list of unique (by key) packages
@@ -31,8 +34,6 @@ const (
 type PackageList struct {
 	// Straight list of packages as map
 	packages map[string]*Package
-	// Has index been prepared?
-	indexed bool
 	// Indexed list of packages, sorted by name internally
 	packagesIndex []*Package
 	// Map of packages for each virtual package (provides)
@@ -41,6 +42,8 @@ type PackageList struct {
 	keyFunc func(p *Package) string
 	// Allow duplicates?
 	duplicatesAllowed bool
+	// Has index been prepared?
+	indexed bool
 }
 
 // PackageConflictError means that package can't be added to the list due to error
@@ -235,7 +238,7 @@ func (l *PackageList) Remove(p *Package) {
 func (l *PackageList) Architectures(includeSource bool) (result []string) {
 	result = make([]string, 0, 10)
 	for _, pkg := range l.packages {
-		if pkg.Architecture != "all" && (pkg.Architecture != "source" || includeSource) && !utils.StrSliceHasItem(result, pkg.Architecture) {
+		if pkg.Architecture != ArchitectureAll && (pkg.Architecture != ArchitectureSource || includeSource) && !utils.StrSliceHasItem(result, pkg.Architecture) {
 			result = append(result, pkg.Architecture)
 		}
 	}
@@ -346,6 +349,14 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 		progress.ShutdownBar()
 	}
 
+	if options&DepVerboseResolve == DepVerboseResolve && progress != nil {
+		missingStr := make([]string, len(missing))
+		for i := range missing {
+			missingStr[i] = missing[i].String()
+		}
+		progress.ColoredPrintf("@{y}Missing dependencies:@| %s", strings.Join(missingStr, ", "))
+	}
+
 	return missing, nil
 }
 
@@ -462,6 +473,11 @@ func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*
 
 // Filter filters package index by specified queries (ORed together), possibly pulling dependencies
 func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
+	return l.FilterWithProgress(queries, withDependencies, source, dependencyOptions, architecturesList, nil)
+}
+
+// FilterWithProgress filters package index by specified queries (ORed together), possibly pulling dependencies and displays progress
+func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string, progress aptly.Progress) (*PackageList, error) {
 	if !l.indexed {
 		panic("list not indexed, can't filter")
 	}
@@ -488,7 +504,7 @@ func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, sour
 			added = 0
 
 			// find missing dependencies
-			missing, err := result.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, nil)
+			missing, err := result.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, progress)
 			if err != nil {
 				return nil, err
 			}
@@ -501,9 +517,12 @@ func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, sour
 					continue
 				}
 
-				searchResults := l.Search(dep, false)
+				searchResults := l.Search(dep, true)
 				if searchResults != nil {
 					for _, p := range searchResults {
+						if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
+							progress.ColoredPrintf("@{g}Injecting package@|: %s", p)
+						}
 						result.Add(p)
 						dependencySource.Add(p)
 						added++
@@ -511,6 +530,11 @@ func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, sour
 							break
 						}
 					}
+				} else {
+					if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
+						progress.ColoredPrintf("@{r}Unsatisfied dependency@|: %s", dep.String())
+					}
+
 				}
 			}
 		}

deb/local_test.go

@@ -18,7 +18,7 @@ type LocalRepoSuite struct {
 var _ = Suite(&LocalRepoSuite{})
 
 func (s *LocalRepoSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.list = NewPackageList()
 	s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
 	s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
@@ -83,7 +83,7 @@ type LocalRepoCollectionSuite struct {
 var _ = Suite(&LocalRepoCollectionSuite{})
 
 func (s *LocalRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collection = NewLocalRepoCollection(s.db)
 
 	s.list = NewPackageList()
@@ -98,14 +98,14 @@ func (s *LocalRepoCollectionSuite) TearDownTest(c *C) {
 }
 
 func (s *LocalRepoCollectionSuite) TestAddByName(c *C) {
-	r, err := s.collection.ByName("local1")
+	_, err := s.collection.ByName("local1")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo := NewLocalRepo("local1", "Comment 1")
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
-	r, err = s.collection.ByName("local1")
+	r, err := s.collection.ByName("local1")
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 
@@ -116,13 +116,13 @@ func (s *LocalRepoCollectionSuite) TestAddByName(c *C) {
 }
 
 func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
-	r, err := s.collection.ByUUID("some-uuid")
+	_, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo := NewLocalRepo("local1", "Comment 1")
 	c.Assert(s.collection.Add(repo), IsNil)
 
-	r, err = s.collection.ByUUID(repo.UUID)
+	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 }

deb/package.go

@@ -24,12 +24,12 @@ type Package struct {
 	Source string
 	// List of virtual packages this package provides
 	Provides []string
+	// Hash of files section
+	FilesHash uint64
 	// Is this source package
 	IsSource bool
 	// Is this udeb package
 	IsUdeb bool
-	// Hash of files section
-	FilesHash uint64
 	// Is this >= 0.6 package?
 	V06Plus bool
 	// Offload fields
@@ -41,6 +41,20 @@ type Package struct {
 	collection *PackageCollection
 }
 
+// Package types
+const (
+	PackageTypeBinary = "deb"
+	PackageTypeUdeb   = "udeb"
+	PackageTypeSource = "source"
+)
+
+// Special arhictectures
+const (
+	ArchitectureAll    = "all"
+	ArhictectureAny    = "any"
+	ArchitectureSource = "source"
+)
+
 // Check interface
 var (
 	_ json.Marshaler = &Package{}
@@ -218,12 +232,12 @@ func (p *Package) GetField(name string) string {
 		return p.Architecture
 	case "$PackageType":
 		if p.IsSource {
-			return "source"
+			return PackageTypeSource
 		}
 		if p.IsUdeb {
-			return "udeb"
+			return PackageTypeUdeb
 		}
-		return "deb"
+		return PackageTypeBinary
 	case "Name":
 		return p.Name
 	case "Version":
@@ -256,7 +270,7 @@ func (p *Package) GetField(name string) string {
 
 // MatchesArchitecture checks whether packages matches specified architecture
 func (p *Package) MatchesArchitecture(arch string) bool {
-	if p.Architecture == "all" && arch != "source" {
+	if p.Architecture == ArchitectureAll && arch != ArchitectureSource {
 		return true
 	}
 
@@ -344,7 +358,7 @@ func (p *Package) GetDependencies(options int) (dependencies []string) {
 		if source == "" {
 			source = p.Name
 		}
-		if strings.Index(source, ")") != -1 {
+		if strings.Contains(source, ")") {
 			dependencies = append(dependencies, fmt.Sprintf("%s {source}", source))
 		} else {
 			dependencies = append(dependencies, fmt.Sprintf("%s (= %s) {source}", source, p.Version))
@@ -403,32 +417,47 @@ func (p *Package) Files() PackageFiles {
 }
 
 // Contents returns cached package contents
-func (p *Package) Contents(packagePool aptly.PackagePool) []string {
+func (p *Package) Contents(packagePool aptly.PackagePool, progress aptly.Progress) []string {
 	if p.IsSource {
 		return nil
 	}
 
-	return p.collection.loadContents(p, packagePool)
+	return p.collection.loadContents(p, packagePool, progress)
 }
 
 // CalculateContents looks up contents in package file
-func (p *Package) CalculateContents(packagePool aptly.PackagePool) []string {
+func (p *Package) CalculateContents(packagePool aptly.PackagePool, progress aptly.Progress) ([]string, error) {
 	if p.IsSource {
-		return nil
+		return nil, nil
 	}
 
 	file := p.Files()[0]
-	path, err := packagePool.Path(file.Filename, file.Checksums.MD5)
+	poolPath, err := file.GetPoolPath(packagePool)
 	if err != nil {
-		panic(err)
+		if progress != nil {
+			progress.ColoredPrintf("@y[!]@| @!Failed to build pool path: @| %s", err)
+		}
+		return nil, err
 	}
 
-	contents, err := GetContentsFromDeb(path)
+	reader, err := packagePool.Open(poolPath)
 	if err != nil {
-		panic(err)
+		if progress != nil {
+			progress.ColoredPrintf("@y[!]@| @!Failed to open package in pool: @| %s", err)
+		}
+		return nil, err
+	}
+	defer reader.Close()
+
+	contents, err := GetContentsFromDeb(reader, file.Filename)
+	if err != nil {
+		if progress != nil {
+			progress.ColoredPrintf("@y[!]@| @!Failed to generate package contents: @| %s", err)
+		}
+		return nil, err
 	}
 
-	return contents
+	return contents, nil
 }
 
 // UpdateFiles saves new state of files
@@ -541,7 +570,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 	}
 
 	for i, f := range p.Files() {
-		sourcePath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
+		sourcePoolPath, err := f.GetPoolPath(packagePool)
 		if err != nil {
 			return err
 		}
@@ -549,7 +578,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 		relPath := filepath.Join("pool", component, poolDir)
 		publishedDirectory := filepath.Join(prefix, relPath)
 
-		err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePath, f.Checksums.MD5, force)
+		err = publishedStorage.LinkFromPool(publishedDirectory, f.Filename, packagePool, sourcePoolPath, f.Checksums, force)
 		if err != nil {
 			return err
 		}
@@ -590,29 +619,25 @@ func (p *Package) PoolDirectory() (string, error) {
 
 // PackageDownloadTask is a element of download queue for the package
 type PackageDownloadTask struct {
-	RepoURI         string
-	DestinationPath string
-	Checksums       utils.ChecksumInfo
+	File         *PackageFile
+	Additional   []PackageDownloadTask
+	TempDownPath string
 }
 
 // DownloadList returns list of missing package files for download in format
 // [[srcpath, dstpath]]
-func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageDownloadTask, err error) {
+func (p *Package) DownloadList(packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage) (result []PackageDownloadTask, err error) {
 	result = make([]PackageDownloadTask, 0, 1)
 
-	for _, f := range p.Files() {
-		poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
-		if err != nil {
-			return nil, err
-		}
-
-		verified, err := f.Verify(packagePool)
+	files := p.Files()
+	for idx := range files {
+		verified, err := files[idx].Verify(packagePool, checksumStorage)
 		if err != nil {
 			return nil, err
 		}
 
 		if !verified {
-			result = append(result, PackageDownloadTask{RepoURI: f.DownloadURL(), DestinationPath: poolPath, Checksums: f.Checksums})
+			result = append(result, PackageDownloadTask{File: &files[idx]})
 		}
 	}
 
@@ -620,11 +645,11 @@ func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageD
 }
 
 // VerifyFiles verifies that all package files have neen correctly downloaded
-func (p *Package) VerifyFiles(packagePool aptly.PackagePool) (result bool, err error) {
+func (p *Package) VerifyFiles(packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage) (result bool, err error) {
 	result = true
 
 	for _, f := range p.Files() {
-		result, err = f.Verify(packagePool)
+		result, err = f.Verify(packagePool, checksumStorage)
 		if err != nil || !result {
 			return
 		}
@@ -639,7 +664,7 @@ func (p *Package) FilepathList(packagePool aptly.PackagePool) ([]string, error)
 	result := make([]string, len(p.Files()))
 
 	for i, f := range p.Files() {
-		result[i], err = packagePool.RelativePath(f.Filename, f.Checksums.MD5)
+		result[i], err = f.GetPoolPath(packagePool)
 		if err != nil {
 			return nil, err
 		}

deb/package_collection.go

@@ -163,7 +163,7 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 }
 
 // loadContents loads or calculates and saves package contents
-func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.PackagePool) []string {
+func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.PackagePool, progress aptly.Progress) []string {
 	encoded, err := collection.db.Get(p.Key("xC"))
 	if err == nil {
 		contents := []string{}
@@ -181,7 +181,11 @@ func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.
 		panic("unable to load contents")
 	}
 
-	contents := p.CalculateContents(packagePool)
+	contents, err := p.CalculateContents(packagePool, progress)
+	if err != nil {
+		// failed to acquire contents, don't persist it
+		return contents
+	}
 
 	var buf bytes.Buffer
 	err = codec.NewEncoder(&buf, collection.codecHandle).Encode(contents)

deb/package_collection_test.go

@@ -17,7 +17,7 @@ var _ = Suite(&PackageCollectionSuite{})
 
 func (s *PackageCollectionSuite) SetUpTest(c *C) {
 	s.p = NewPackageFromControlFile(packageStanza.Copy())
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collection = NewPackageCollection(s.db)
 }
 

deb/package_files.go

@@ -4,7 +4,6 @@ import (
 	"encoding/binary"
 	"fmt"
 	"hash/fnv"
-	"os"
 	"path/filepath"
 	"sort"
 	"strconv"
@@ -20,25 +19,33 @@ type PackageFile struct {
 	Filename string
 	// Hashes for the file
 	Checksums utils.ChecksumInfo
+	// PoolPath persists relative path to file in the package pool
+	PoolPath string
 	// Temporary field used while downloading, stored relative path on the mirror
 	downloadPath string
 }
 
 // Verify that package file is present and correct
-func (f *PackageFile) Verify(packagePool aptly.PackagePool) (bool, error) {
-	poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
-	if err != nil {
-		return false, err
+func (f *PackageFile) Verify(packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage) (bool, error) {
+	generatedPoolPath, exists, err := packagePool.Verify(f.PoolPath, f.Filename, &f.Checksums, checksumStorage)
+	if exists && err == nil {
+		f.PoolPath = generatedPoolPath
 	}
 
-	st, err := os.Stat(poolPath)
-	if err != nil {
-		return false, nil
+	return exists, err
+}
+
+// GetPoolPath returns path to the file in the pool
+//
+// For legacy packages which do not have PoolPath field set, that calculates LegacyPath via pool
+func (f *PackageFile) GetPoolPath(packagePool aptly.PackagePool) (string, error) {
+	var err error
+
+	if f.PoolPath == "" {
+		f.PoolPath, err = packagePool.LegacyPath(f.Filename, &f.Checksums)
 	}
 
-	// verify size
-	// TODO: verify checksum if configured
-	return st.Size() == f.Checksums.Size, nil
+	return f.PoolPath, err
 }
 
 // DownloadURL return relative URL to package download location

deb/package_files_test.go

@@ -1,9 +1,10 @@
 package deb
 
 import (
-	"os"
+	"io/ioutil"
 	"path/filepath"
 
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/utils"
 
@@ -12,11 +13,13 @@ import (
 
 type PackageFilesSuite struct {
 	files PackageFiles
+	cs    aptly.ChecksumStorage
 }
 
 var _ = Suite(&PackageFilesSuite{})
 
 func (s *PackageFilesSuite) SetUpTest(c *C) {
+	s.cs = files.NewMockChecksumStorage()
 	s.files = PackageFiles{PackageFile{
 		Filename:     "alien-arena-common_7.40-2_i386.deb",
 		downloadPath: "pool/contrib/a/alien-arena",
@@ -29,27 +32,24 @@ func (s *PackageFilesSuite) SetUpTest(c *C) {
 }
 
 func (s *PackageFilesSuite) TestVerify(c *C) {
-	packagePool := files.NewPackagePool(c.MkDir())
-	poolPath, _ := packagePool.Path(s.files[0].Filename, s.files[0].Checksums.MD5)
+	packagePool := files.NewPackagePool(c.MkDir(), false)
 
-	result, err := s.files[0].Verify(packagePool)
+	result, err := s.files[0].Verify(packagePool, s.cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, false)
 
-	err = os.MkdirAll(filepath.Dir(poolPath), 0755)
-	c.Assert(err, IsNil)
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
+	c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
 
-	file, err := os.Create(poolPath)
-	c.Assert(err, IsNil)
-	file.WriteString("abcde")
-	file.Close()
+	s.files[0].PoolPath, _ = packagePool.Import(tmpFilepath, s.files[0].Filename, &s.files[0].Checksums, false, s.cs)
 
-	result, err = s.files[0].Verify(packagePool)
+	s.files[0].Checksums.Size = 187518
+	result, err = s.files[0].Verify(packagePool, s.cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, false)
 
 	s.files[0].Checksums.Size = 5
-	result, err = s.files[0].Verify(packagePool)
+	result, err = s.files[0].Verify(packagePool, s.cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, true)
 }

deb/package_test.go

@@ -2,12 +2,11 @@ package deb
 
 import (
 	"bytes"
-	"os"
+	"io/ioutil"
 	"path/filepath"
 	"regexp"
 
 	"github.com/smira/aptly/files"
-	"github.com/smira/aptly/utils"
 
 	. "gopkg.in/check.v1"
 )
@@ -300,7 +299,7 @@ func (s *PackageSuite) TestMatchesDependency(c *C) {
 	// ~
 	c.Check(
 		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
-			Regexp: regexp.MustCompile("7\\.40-.*")}), Equals, true)
+			Regexp: regexp.MustCompile(`7\.40-.*`)}), Equals, true)
 	c.Check(
 		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
 			Regexp: regexp.MustCompile("40")}), Equals, true)
@@ -363,19 +362,17 @@ func (s *PackageSuite) TestPoolDirectory(c *C) {
 }
 
 func (s *PackageSuite) TestLinkFromPool(c *C) {
-	packagePool := files.NewPackagePool(c.MkDir())
-	publishedStorage := files.NewPublishedStorage(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir(), false)
+	cs := files.NewMockChecksumStorage()
+	publishedStorage := files.NewPublishedStorage(c.MkDir(), "", "")
 	p := NewPackageFromControlFile(s.stanza)
 
-	poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5)
-	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
-	c.Assert(err, IsNil)
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
+	c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
 
-	file, err := os.Create(poolPath)
-	c.Assert(err, IsNil)
-	file.Close()
+	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
 
-	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
+	err := p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
 	c.Check(err, IsNil)
 	c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
 	c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena")
@@ -387,7 +384,7 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
 }
 
 func (s *PackageSuite) TestFilepathList(c *C) {
-	packagePool := files.NewPackagePool(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir(), true)
 	p := NewPackageFromControlFile(s.stanza)
 
 	list, err := p.FilepathList(packagePool)
@@ -396,31 +393,24 @@ func (s *PackageSuite) TestFilepathList(c *C) {
 }
 
 func (s *PackageSuite) TestDownloadList(c *C) {
-	packagePool := files.NewPackagePool(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir(), false)
+	cs := files.NewMockChecksumStorage()
 	p := NewPackageFromControlFile(s.stanza)
 	p.Files()[0].Checksums.Size = 5
-	poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5)
 
-	list, err := p.DownloadList(packagePool)
+	list, err := p.DownloadList(packagePool, cs)
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []PackageDownloadTask{
 		{
-			RepoURI:         "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb",
-			DestinationPath: poolPath,
-			Checksums: utils.ChecksumInfo{Size: 5,
-				MD5:    "1e8cba92c41420aa7baa8a5718d67122",
-				SHA1:   "46955e48cad27410a83740a21d766ce362364024",
-				SHA256: "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5"}}})
+			File: &p.Files()[0],
+		},
+	})
 
-	err = os.MkdirAll(filepath.Dir(poolPath), 0755)
-	c.Assert(err, IsNil)
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
+	c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
+	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
 
-	file, err := os.Create(poolPath)
-	c.Assert(err, IsNil)
-	file.WriteString("abcde")
-	file.Close()
-
-	list, err = p.DownloadList(packagePool)
+	list, err = p.DownloadList(packagePool, cs)
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []PackageDownloadTask{})
 }
@@ -428,24 +418,22 @@ func (s *PackageSuite) TestDownloadList(c *C) {
 func (s *PackageSuite) TestVerifyFiles(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
-	packagePool := files.NewPackagePool(c.MkDir())
-	poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5)
+	packagePool := files.NewPackagePool(c.MkDir(), false)
+	cs := files.NewMockChecksumStorage()
 
-	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
-	c.Assert(err, IsNil)
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
+	c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
 
-	file, err := os.Create(poolPath)
-	c.Assert(err, IsNil)
-	file.WriteString("abcde")
-	file.Close()
+	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
 
-	result, err := p.VerifyFiles(packagePool)
+	p.Files()[0].Checksums.Size = 100
+	result, err := p.VerifyFiles(packagePool, cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, false)
 
 	p.Files()[0].Checksums.Size = 5
 
-	result, err = p.VerifyFiles(packagePool)
+	result, err = p.VerifyFiles(packagePool, cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, true)
 }

deb/publish.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -36,17 +37,17 @@ type PublishedRepo struct {
 	// Internal unique ID
 	UUID string
 	// Storage & Prefix & distribution should be unique across all published repositories
-	Storage      string
-	Prefix       string
-	Distribution string
-	Origin       string
-	Label        string
+	Storage              string
+	Prefix               string
+	Distribution         string
+	Origin               string
+	NotAutomatic         string
+	ButAutomaticUpgrades string
+	Label                string
 	// Architectures is a list of all architectures published
 	Architectures []string
 	// SourceKind is "local"/"repo"
 	SourceKind string
-	// Skip contents generation
-	SkipContents bool
 
 	// Map of sources by each component: component name -> source UUID
 	Sources map[string]string
@@ -55,10 +56,12 @@ type PublishedRepo struct {
 	Component string
 	// SourceUUID is UUID of either snapshot or local repo
 	SourceUUID string `codec:"SnapshotUUID"`
-
 	// Map of component to source items
 	sourceItems map[string]repoSourceItem
 
+	// Skip contents generation
+	SkipContents bool
+
 	// True if repo is being re-published
 	rePublishing bool
 }
@@ -75,6 +78,7 @@ func ParsePrefix(param string) (storage, prefix string) {
 	} else {
 		prefix = param
 	}
+	prefix = strings.TrimPrefix(strings.TrimSuffix(prefix, "/"), "/")
 	return
 }
 
@@ -96,19 +100,19 @@ func walkUpTree(source interface{}, collectionFactory *CollectionFactory) (rootD
 
 		if snapshot, ok := head.(*Snapshot); ok {
 			for _, uuid := range snapshot.SourceIDs {
-				if snapshot.SourceKind == "repo" {
+				if snapshot.SourceKind == SourceRemoteRepo {
 					remoteRepo, err := collectionFactory.RemoteRepoCollection().ByUUID(uuid)
 					if err != nil {
 						continue
 					}
 					current = append(current, remoteRepo)
-				} else if snapshot.SourceKind == "local" {
+				} else if snapshot.SourceKind == SourceLocalRepo {
 					localRepo, err := collectionFactory.LocalRepoCollection().ByUUID(uuid)
 					if err != nil {
 						continue
 					}
 					current = append(current, localRepo)
-				} else if snapshot.SourceKind == "snapshot" {
+				} else if snapshot.SourceKind == SourceSnapshot {
 					snap, err := collectionFactory.SnapshotCollection().ByUUID(uuid)
 					if err != nil {
 						continue
@@ -166,23 +170,20 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 		component               string
 		snapshot                *Snapshot
 		localRepo               *LocalRepo
-		ok                      bool
+		fields                  = make(map[string][]string)
 	)
 
 	// get first source
 	source = sources[0]
 
 	// figure out source kind
-	snapshot, ok = source.(*Snapshot)
-	if ok {
-		result.SourceKind = "snapshot"
-	} else {
-		localRepo, ok = source.(*LocalRepo)
-		if ok {
-			result.SourceKind = "local"
-		} else {
-			panic("unknown source kind")
-		}
+	switch source.(type) {
+	case *Snapshot:
+		result.SourceKind = SourceSnapshot
+	case *LocalRepo:
+		result.SourceKind = SourceLocalRepo
+	default:
+		panic("unknown source kind")
 	}
 
 	for i := range sources {
@@ -213,11 +214,21 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 			return nil, fmt.Errorf("duplicate component name: %s", component)
 		}
 
-		if result.SourceKind == "snapshot" {
+		if result.SourceKind == SourceSnapshot {
 			snapshot = source.(*Snapshot)
 			result.Sources[component] = snapshot.UUID
 			result.sourceItems[component] = repoSourceItem{snapshot: snapshot}
-		} else if result.SourceKind == "local" {
+
+			if !utils.StrSliceHasItem(fields["Origin"], snapshot.Origin) {
+				fields["Origin"] = append(fields["Origin"], snapshot.Origin)
+			}
+			if !utils.StrSliceHasItem(fields["NotAutomatic"], snapshot.NotAutomatic) {
+				fields["NotAutomatic"] = append(fields["NotAutomatic"], snapshot.NotAutomatic)
+			}
+			if !utils.StrSliceHasItem(fields["ButAutomaticUpgrades"], snapshot.ButAutomaticUpgrades) {
+				fields["ButAutomaticUpgrades"] = append(fields["ButAutomaticUpgrades"], snapshot.ButAutomaticUpgrades)
+			}
+		} else if result.SourceKind == SourceLocalRepo {
 			localRepo = source.(*LocalRepo)
 			result.Sources[component] = localRepo.UUID
 			result.sourceItems[component] = repoSourceItem{localRepo: localRepo, packageRefs: localRepo.RefList()}
@@ -226,12 +237,7 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 
 	// clean & verify prefix
 	prefix = filepath.Clean(prefix)
-	if strings.HasPrefix(prefix, "/") {
-		prefix = prefix[1:]
-	}
-	if strings.HasSuffix(prefix, "/") {
-		prefix = prefix[:len(prefix)-1]
-	}
+	prefix = strings.TrimPrefix(strings.TrimSuffix(prefix, "/"), "/")
 	prefix = filepath.Clean(prefix)
 
 	for _, part := range strings.Split(prefix, "/") {
@@ -252,12 +258,23 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 		}
 	}
 
-	if strings.Index(distribution, "/") != -1 {
+	if strings.Contains(distribution, "/") {
 		return nil, fmt.Errorf("invalid distribution %s, '/' is not allowed", distribution)
 	}
 
 	result.Distribution = distribution
 
+	// only fields which are unique by all given snapshots are set on published
+	if len(fields["Origin"]) == 1 {
+		result.Origin = fields["Origin"][0]
+	}
+	if len(fields["NotAutomatic"]) == 1 {
+		result.NotAutomatic = fields["NotAutomatic"][0]
+	}
+	if len(fields["ButAutomaticUpgrades"]) == 1 {
+		result.ButAutomaticUpgrades = fields["ButAutomaticUpgrades"][0]
+	}
+
 	return result, nil
 }
 
@@ -284,15 +301,17 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
 	}
 
 	return json.Marshal(map[string]interface{}{
-		"Architectures": p.Architectures,
-		"Distribution":  p.Distribution,
-		"Label":         p.Label,
-		"Origin":        p.Origin,
-		"Prefix":        p.Prefix,
-		"SourceKind":    p.SourceKind,
-		"Sources":       sources,
-		"Storage":       p.Storage,
-		"SkipContents":  p.SkipContents,
+		"Architectures":        p.Architectures,
+		"Distribution":         p.Distribution,
+		"Label":                p.Label,
+		"Origin":               p.Origin,
+		"NotAutomatic":         p.NotAutomatic,
+		"ButAutomaticUpgrades": p.ButAutomaticUpgrades,
+		"Prefix":               p.Prefix,
+		"SourceKind":           p.SourceKind,
+		"Sources":              sources,
+		"Storage":              p.Storage,
+		"SkipContents":         p.SkipContents,
 	})
 }
 
@@ -315,19 +334,27 @@ func (p *PublishedRepo) String() string {
 		sources = append(sources, fmt.Sprintf("{%s: %s}", component, source))
 	}
 
+	var extras []string
 	var extra string
 
 	if p.Origin != "" {
-		extra += fmt.Sprintf("origin: %s", p.Origin)
+		extras = append(extras, fmt.Sprintf("origin: %s", p.Origin))
+	}
+
+	if p.NotAutomatic != "" {
+		extras = append(extras, fmt.Sprintf("notautomatic: %s", p.NotAutomatic))
+	}
+
+	if p.ButAutomaticUpgrades != "" {
+		extras = append(extras, fmt.Sprintf("butautomaticupgrades: %s", p.ButAutomaticUpgrades))
 	}
 
 	if p.Label != "" {
-		if extra != "" {
-			extra += ", "
-		}
-		extra += fmt.Sprintf("label: %s", p.Label)
+		extras = append(extras, fmt.Sprintf("label: %s", p.Label))
 	}
 
+	extra = strings.Join(extras, ", ")
+
 	if extra != "" {
 		extra = " (" + extra + ")"
 	}
@@ -358,10 +385,10 @@ func (p *PublishedRepo) RefKey(component string) []byte {
 // RefList returns list of package refs in local repo
 func (p *PublishedRepo) RefList(component string) *PackageRefList {
 	item := p.sourceItems[component]
-	if p.SourceKind == "local" {
+	if p.SourceKind == SourceLocalRepo {
 		return item.packageRefs
 	}
-	if p.SourceKind == "snapshot" {
+	if p.SourceKind == SourceSnapshot {
 		return item.snapshot.RefList()
 	}
 	panic("unknown source")
@@ -380,7 +407,7 @@ func (p *PublishedRepo) Components() []string {
 
 // UpdateLocalRepo updates content from local repo in component
 func (p *PublishedRepo) UpdateLocalRepo(component string) {
-	if p.SourceKind != "local" {
+	if p.SourceKind != SourceLocalRepo {
 		panic("not local repo publish")
 	}
 
@@ -393,7 +420,7 @@ func (p *PublishedRepo) UpdateLocalRepo(component string) {
 
 // UpdateSnapshot switches snapshot for component
 func (p *PublishedRepo) UpdateSnapshot(component string, snapshot *Snapshot) {
-	if p.SourceKind != "snapshot" {
+	if p.SourceKind != SourceSnapshot {
 		panic("not snapshot publish")
 	}
 
@@ -425,7 +452,7 @@ func (p *PublishedRepo) Decode(input []byte) error {
 
 	// old PublishedRepo were publishing only snapshots
 	if p.SourceKind == "" {
-		p.SourceKind = "snapshot"
+		p.SourceKind = SourceSnapshot
 	}
 
 	// <0.6 aptly used single SourceUUID + Component instead of Sources
@@ -456,7 +483,7 @@ func (p *PublishedRepo) GetLabel() string {
 
 // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
 func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
-	collectionFactory *CollectionFactory, signer utils.Signer, progress aptly.Progress, forceOverwrite bool) error {
+	collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool) error {
 	publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage)
 
 	err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
@@ -474,8 +501,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		return err
 	}
 	defer func() {
-		var e error
-		e = tempDB.Close()
+		e := tempDB.Close()
 		if e != nil && progress != nil {
 			progress.Printf("failed to close temp DB: %s", err)
 		}
@@ -583,7 +609,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 							contentIndexes[key] = contentIndex
 						}
 
-						contentIndex.Push(pkg, packagePool)
+						contentIndex.Push(pkg, packagePool, progress)
 					}
 
 					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
@@ -621,7 +647,8 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 					continue
 				}
 
-				bufWriter, err := indexes.ContentsIndex(component, arch, udeb).BufWriter()
+				var bufWriter *bufio.Writer
+				bufWriter, err = indexes.ContentsIndex(component, arch, udeb).BufWriter()
 				if err != nil {
 					return fmt.Errorf("unable to generate contents index: %v", err)
 				}
@@ -682,11 +709,17 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 	release := make(Stanza)
 	release["Origin"] = p.GetOrigin()
+	if p.NotAutomatic != "" {
+		release["NotAutomatic"] = p.NotAutomatic
+	}
+	if p.ButAutomaticUpgrades != "" {
+		release["ButAutomaticUpgrades"] = p.ButAutomaticUpgrades
+	}
 	release["Label"] = p.GetLabel()
 	release["Suite"] = p.Distribution
 	release["Codename"] = p.Distribution
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
-	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{"source"}), " ")
+	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
 	release["Description"] = " Generated by aptly\n"
 	release["MD5Sum"] = ""
 	release["SHA1"] = ""
@@ -730,12 +763,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		return err
 	}
 
-	err = indexes.RenameFiles()
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return indexes.RenameFiles()
 }
 
 // RemoveFiles removes files that were created by Publish
@@ -834,7 +862,7 @@ func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) (err erro
 		return
 	}
 
-	if repo.SourceKind == "local" {
+	if repo.SourceKind == SourceLocalRepo {
 		for component, item := range repo.sourceItems {
 			err = collection.db.Put(repo.RefKey(component), item.packageRefs.Encode())
 			if err != nil {
@@ -849,7 +877,7 @@ func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) (err erro
 func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, collectionFactory *CollectionFactory) (err error) {
 	repo.sourceItems = make(map[string]repoSourceItem)
 
-	if repo.SourceKind == "snapshot" {
+	if repo.SourceKind == SourceSnapshot {
 		for component, sourceUUID := range repo.Sources {
 			item := repoSourceItem{}
 
@@ -864,7 +892,7 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
 
 			repo.sourceItems[component] = item
 		}
-	} else if repo.SourceKind == "local" {
+	} else if repo.SourceKind == SourceLocalRepo {
 		for component, sourceUUID := range repo.Sources {
 			item := repoSourceItem{}
 
@@ -932,7 +960,7 @@ func (collection *PublishedRepoCollection) ByUUID(uuid string) (*PublishedRepo,
 func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*PublishedRepo {
 	var result []*PublishedRepo
 	for _, r := range collection.list {
-		if r.SourceKind == "snapshot" {
+		if r.SourceKind == SourceSnapshot {
 			if r.SourceUUID == snapshot.UUID {
 				result = append(result, r)
 			}
@@ -952,7 +980,7 @@ func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*Pub
 func (collection *PublishedRepoCollection) ByLocalRepo(repo *LocalRepo) []*PublishedRepo {
 	var result []*PublishedRepo
 	for _, r := range collection.list {
-		if r.SourceKind == "local" {
+		if r.SourceKind == SourceLocalRepo {
 			if r.SourceUUID == repo.UUID {
 				result = append(result, r)
 			}

deb/publish_test.go

@@ -74,6 +74,7 @@ type PublishedRepoSuite struct {
 	provider                            *FakeStorageProvider
 	publishedStorage, publishedStorage2 *files.PublishedStorage
 	packagePool                         aptly.PackagePool
+	cs                                  aptly.ChecksumStorage
 	localRepo                           *LocalRepo
 	snapshot, snapshot2                 *Snapshot
 	db                                  database.Storage
@@ -86,17 +87,31 @@ var _ = Suite(&PublishedRepoSuite{})
 func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.SetUpPackages()
 
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.factory = NewCollectionFactory(s.db)
 
 	s.root = c.MkDir()
-	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.publishedStorage = files.NewPublishedStorage(s.root, "", "")
 	s.root2 = c.MkDir()
-	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2, "", "")
 	s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{
 		"":            s.publishedStorage,
 		"files:other": s.publishedStorage2}}
-	s.packagePool = files.NewPackagePool(s.root)
+	s.packagePool = files.NewPackagePool(s.root, false)
+	s.cs = files.NewMockChecksumStorage()
+
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
+	c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
+
+	var err error
+	s.p1.Files()[0].PoolPath, err = s.packagePool.Import(tmpFilepath, s.p1.Files()[0].Filename, &s.p1.Files()[0].Checksums, false, s.cs)
+	c.Assert(err, IsNil)
+
+	s.p1.UpdateFiles(s.p1.Files())
+	s.p2.UpdateFiles(s.p1.Files())
+	s.p3.UpdateFiles(s.p1.Files())
+
+	s.reflist = NewPackageRefListFromPackageList(s.list)
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	repo.packageRefs = s.reflist
@@ -131,12 +146,6 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 	s.repo5.SkipContents = true
-
-	poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
-	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
-	f, err := os.Create(poolPath)
-	c.Assert(err, IsNil)
-	f.Close()
 }
 
 func (s *PublishedRepoSuite) TearDownTest(c *C) {
@@ -268,7 +277,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"main"})
 
-	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	_, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 	c.Check(err, ErrorMatches, "unable to guess distribution name, please specify explicitly")
 
 	s.localRepo.DefaultDistribution = "precise"
@@ -292,7 +301,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"contrib", "main"})
 
-	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	_, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
 	c.Check(err, ErrorMatches, "duplicate component name: main")
 }
 
@@ -440,7 +449,7 @@ type PublishedRepoCollectionSuite struct {
 var _ = Suite(&PublishedRepoCollectionSuite{})
 
 func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.factory = NewCollectionFactory(s.db)
 
 	s.snapshotCollection = s.factory.SnapshotCollection()
@@ -468,7 +477,7 @@ func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) {
 }
 
 func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C) {
-	r, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	_, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	c.Assert(s.collection.Add(s.repo1), IsNil)
@@ -480,7 +489,7 @@ func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C)
 	c.Assert(s.collection.Add(s.repo4), IsNil)
 	c.Assert(s.collection.Add(s.repo5), IsNil)
 
-	r, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	r, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, IsNil)
 
 	err = s.collection.LoadComplete(r, s.factory)
@@ -496,16 +505,17 @@ func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C)
 	c.Assert(r.String(), Equals, s.repo1.String())
 
 	r, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "precise")
+	c.Assert(err, IsNil)
 	c.Check(r.String(), Equals, s.repo5.String())
 }
 
 func (s *PublishedRepoCollectionSuite) TestByUUID(c *C) {
-	r, err := s.collection.ByUUID(s.repo1.UUID)
+	_, err := s.collection.ByUUID(s.repo1.UUID)
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	c.Assert(s.collection.Add(s.repo1), IsNil)
 
-	r, err = s.collection.ByUUID(s.repo1.UUID)
+	r, err := s.collection.ByUUID(s.repo1.UUID)
 	c.Assert(err, IsNil)
 
 	err = s.collection.LoadComplete(r, s.factory)
@@ -552,7 +562,7 @@ func (s *PublishedRepoCollectionSuite) TestLoadPre0_6(c *C) {
 		Prefix:        "ppa",
 		Distribution:  "anaconda",
 		Architectures: []string{"i386"},
-		SourceKind:    "local",
+		SourceKind:    SourceLocalRepo,
 		Component:     "contrib",
 		SourceUUID:    s.localRepo.UUID,
 	}
@@ -630,7 +640,7 @@ type PublishedRepoRemoveSuite struct {
 var _ = Suite(&PublishedRepoRemoveSuite{})
 
 func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.factory = NewCollectionFactory(s.db)
 
 	s.snapshotCollection = s.factory.SnapshotCollection()
@@ -653,7 +663,7 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 	s.collection.Add(s.repo5)
 
 	s.root = c.MkDir()
-	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.publishedStorage = files.NewPublishedStorage(s.root, "", "")
 	s.publishedStorage.MkDir("ppa/dists/anaconda")
 	s.publishedStorage.MkDir("ppa/dists/meduza")
 	s.publishedStorage.MkDir("ppa/dists/osminog")
@@ -663,7 +673,7 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 	s.publishedStorage.MkDir("pool/main")
 
 	s.root2 = c.MkDir()
-	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2, "", "")
 	s.publishedStorage2.MkDir("ppa/dists/osminog")
 	s.publishedStorage2.MkDir("ppa/pool/contrib")
 

deb/query.go

@@ -204,7 +204,7 @@ func (q *FieldQuery) Fast(list PackageCatalog) bool {
 // String interface
 func (q *FieldQuery) String() string {
 	escape := func(val string) string {
-		if strings.IndexAny(val, "()|,!{} \t\n") != -1 {
+		if strings.ContainsAny(val, "()|,!{} \t\n") {
 			return "'" + strings.Replace(strings.Replace(val, "\\", "\\\\", -1), "'", "\\'", -1) + "'"
 		}
 		return val

deb/reflist.go

@@ -92,7 +92,7 @@ func (l *PackageRefList) Has(p *Package) bool {
 	key := p.Key("")
 
 	i := sort.Search(len(l.Refs), func(j int) bool { return bytes.Compare(l.Refs[j], key) >= 0 })
-	return i < len(l.Refs) && bytes.Compare(l.Refs[i], key) == 0
+	return i < len(l.Refs) && bytes.Equal(l.Refs[i], key)
 }
 
 // Strings builds list of strings with package keys
@@ -395,6 +395,4 @@ func (l *PackageRefList) FilterLatestRefs() {
 
 		lastArch, lastName, lastVer = arch, name, ver
 	}
-
-	return
 }

deb/reflist_test.go

@@ -44,7 +44,7 @@ func (s *PackageRefListSuite) SetUpTest(c *C) {
 }
 
 func (s *PackageRefListSuite) TestNewPackageListFromRefList(c *C) {
-	db, _ := database.OpenDB(c.MkDir())
+	db, _ := database.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)
 	coll.Update(s.p1)
 	coll.Update(s.p3)
@@ -166,7 +166,7 @@ func (s *PackageRefListSuite) TestSubstract(c *C) {
 }
 
 func (s *PackageRefListSuite) TestDiff(c *C) {
-	db, _ := database.OpenDB(c.MkDir())
+	db, _ := database.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)
 
 	packages := []*Package{
@@ -238,7 +238,7 @@ func (s *PackageRefListSuite) TestDiff(c *C) {
 }
 
 func (s *PackageRefListSuite) TestMerge(c *C) {
-	db, _ := database.OpenDB(c.MkDir())
+	db, _ := database.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)
 
 	packages := []*Package{

deb/remote.go

@@ -7,7 +7,6 @@ import (
 	"net/url"
 	"os"
 	"path"
-	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
@@ -18,6 +17,7 @@ import (
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
@@ -45,10 +45,6 @@ type RemoteRepo struct {
 	Components []string
 	// List of architectures to fetch, if empty, then fetch all architectures
 	Architectures []string
-	// Should we download sources?
-	DownloadSources bool
-	// Should we download .udebs?
-	DownloadUdebs bool
 	// Meta-information about repository
 	Meta Stanza
 	// Last update date
@@ -57,20 +53,22 @@ type RemoteRepo struct {
 	ReleaseFiles map[string]utils.ChecksumInfo
 	// Filter for packages
 	Filter string
+	// Status marks state of repository (being updated, no action)
+	Status int
+	// WorkerPID is PID of the process modifying the mirror (if any)
+	WorkerPID int
 	// FilterWithDeps to include dependencies from filter query
 	FilterWithDeps bool
 	// SkipComponentCheck skips component list verification
 	SkipComponentCheck bool
 	// SkipArchitectureCheck skips architecture list verification
 	SkipArchitectureCheck bool
-	// Status marks state of repository (being updated, no action)
-	Status int
-	// WorkerPID is PID of the process modifying the mirror (if any)
-	WorkerPID int
+	// Should we download sources?
+	DownloadSources bool
+	// Should we download .udebs?
+	DownloadUdebs bool
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
-	// Temporary list of package refs
-	tempPackageRefs *PackageRefList
 	// Parsed archived root
 	archiveRootURL *url.URL
 	// Current list of packages (filled while updating mirror)
@@ -193,49 +191,49 @@ func (repo *RemoteRepo) CheckLock() error {
 	return nil
 }
 
-// ReleaseURL returns URL to Release* files in repo root
-func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
+// IndexesRootURL builds URL for various indexes
+func (repo *RemoteRepo) IndexesRootURL() *url.URL {
 	var path *url.URL
 
 	if !repo.IsFlat() {
-		path = &url.URL{Path: fmt.Sprintf("dists/%s/%s", repo.Distribution, name)}
+		path = &url.URL{Path: fmt.Sprintf("dists/%s/", repo.Distribution)}
 	} else {
-		path = &url.URL{Path: filepath.Join(repo.Distribution, name)}
+		path = &url.URL{Path: repo.Distribution}
 	}
 
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
-// FlatBinaryURL returns URL to Packages files for flat repo
-func (repo *RemoteRepo) FlatBinaryURL() *url.URL {
-	path := &url.URL{Path: filepath.Join(repo.Distribution, "Packages")}
-	return repo.archiveRootURL.ResolveReference(path)
+// ReleaseURL returns URL to Release* files in repo root
+func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
+	return repo.IndexesRootURL().ResolveReference(&url.URL{Path: name})
 }
 
-// FlatSourcesURL returns URL to Sources files for flat repo
-func (repo *RemoteRepo) FlatSourcesURL() *url.URL {
-	path := &url.URL{Path: filepath.Join(repo.Distribution, "Sources")}
-	return repo.archiveRootURL.ResolveReference(path)
+// FlatBinaryPath returns path to Packages files for flat repo
+func (repo *RemoteRepo) FlatBinaryPath() string {
+	return "Packages"
 }
 
-// BinaryURL returns URL of Packages files for given component and
+// FlatSourcesPath returns path to Sources files for flat repo
+func (repo *RemoteRepo) FlatSourcesPath() string {
+	return "Sources"
+}
+
+// BinaryPath returns path to Packages files for given component and
 // architecture
-func (repo *RemoteRepo) BinaryURL(component string, architecture string) *url.URL {
-	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/binary-%s/Packages", repo.Distribution, component, architecture)}
-	return repo.archiveRootURL.ResolveReference(path)
+func (repo *RemoteRepo) BinaryPath(component string, architecture string) string {
+	return fmt.Sprintf("%s/binary-%s/Packages", component, architecture)
 }
 
-// SourcesURL returns URL of Sources files for given component
-func (repo *RemoteRepo) SourcesURL(component string) *url.URL {
-	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/source/Sources", repo.Distribution, component)}
-	return repo.archiveRootURL.ResolveReference(path)
+// SourcesPath returns path to Sources files for given component
+func (repo *RemoteRepo) SourcesPath(component string) string {
+	return fmt.Sprintf("%s/source/Sources", component)
 }
 
-// UdebURL returns URL of Packages files for given component and
+// UdebPath returns path of Packages files for given component and
 // architecture
-func (repo *RemoteRepo) UdebURL(component string, architecture string) *url.URL {
-	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/debian-installer/binary-%s/Packages", repo.Distribution, component, architecture)}
-	return repo.archiveRootURL.ResolveReference(path)
+func (repo *RemoteRepo) UdebPath(component string, architecture string) string {
+	return fmt.Sprintf("%s/debian-installer/binary-%s/Packages", component, architecture)
 }
 
 // PackageURL returns URL of package file relative to repository root
@@ -246,7 +244,7 @@ func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
 }
 
 // Fetch updates information about repository
-func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error {
+func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier pgp.Verifier) error {
 	var (
 		release, inrelease, releasesig *os.File
 		err                            error
@@ -292,7 +290,7 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error
 			return err
 		}
 
-		err = verifier.VerifyDetachedSignature(releasesig, release)
+		err = verifier.VerifyDetachedSignature(releasesig, release, true)
 		if err != nil {
 			return err
 		}
@@ -316,7 +314,7 @@ ok:
 		architectures := strings.Split(stanza["Architectures"], " ")
 		sort.Strings(architectures)
 		// "source" architecture is never present, despite Release file claims
-		architectures = utils.StrSlicesSubstract(architectures, []string{"source"})
+		architectures = utils.StrSlicesSubstract(architectures, []string{ArchitectureSource})
 		if len(repo.Architectures) == 0 {
 			repo.Architectures = architectures
 		} else if !repo.SkipArchitectureCheck {
@@ -331,9 +329,7 @@ ok:
 		if strings.Contains(repo.Distribution, "/") {
 			distributionLast := path.Base(repo.Distribution) + "/"
 			for i := range components {
-				if strings.HasPrefix(components[i], distributionLast) {
-					components[i] = components[i][len(distributionLast):]
-				}
+				components[i] = strings.TrimPrefix(components[i], distributionLast)
 			}
 		}
 		if len(repo.Components) == 0 {
@@ -411,30 +407,30 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 	repo.packageList = NewPackageList()
 
 	// Download and parse all Packages & Source files
-	packagesURLs := [][]string{}
+	packagesPaths := [][]string{}
 
 	if repo.IsFlat() {
-		packagesURLs = append(packagesURLs, []string{repo.FlatBinaryURL().String(), "binary"})
+		packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary})
 		if repo.DownloadSources {
-			packagesURLs = append(packagesURLs, []string{repo.FlatSourcesURL().String(), "source"})
+			packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource})
 		}
 	} else {
 		for _, component := range repo.Components {
 			for _, architecture := range repo.Architectures {
-				packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
+				packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary})
 				if repo.DownloadUdebs {
-					packagesURLs = append(packagesURLs, []string{repo.UdebURL(component, architecture).String(), "udeb"})
+					packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb})
 				}
 			}
 			if repo.DownloadSources {
-				packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
+				packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource})
 			}
 		}
 	}
 
-	for _, info := range packagesURLs {
-		url, kind := info[0], info[1]
-		packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch, maxTries)
+	for _, info := range packagesPaths {
+		path, kind := info[0], info[1]
+		packagesReader, packagesFile, err := http.DownloadTryCompression(d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries)
 		if err != nil {
 			return err
 		}
@@ -459,11 +455,11 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 
 			var p *Package
 
-			if kind == "binary" {
+			if kind == PackageTypeBinary {
 				p = NewPackageFromControlFile(stanza)
-			} else if kind == "udeb" {
+			} else if kind == PackageTypeUdeb {
 				p = NewUdebPackageFromControlFile(stanza)
-			} else if kind == "source" {
+			} else if kind == PackageTypeSource {
 				p, err = NewSourcePackageFromControlFile(stanza)
 				if err != nil {
 					return err
@@ -477,11 +473,6 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 					return err
 				}
 			}
-
-			err = collectionFactory.PackageCollection().Update(p)
-			if err != nil {
-				return err
-			}
 		}
 
 		progress.ShutdownBar()
@@ -491,14 +482,14 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 }
 
 // ApplyFilter applies filtering to already built PackageList
-func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQuery) (oldLen, newLen int, err error) {
+func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQuery, progress aptly.Progress) (oldLen, newLen int, err error) {
 	repo.packageList.PrepareIndex()
 
 	emptyList := NewPackageList()
 	emptyList.PrepareIndex()
 
 	oldLen = repo.packageList.Len()
-	repo.packageList, err = repo.packageList.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures)
+	repo.packageList, err = repo.packageList.FilterWithProgress([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures, progress)
 	if repo.packageList != nil {
 		newLen = repo.packageList.Len()
 	}
@@ -506,24 +497,40 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu
 }
 
 // BuildDownloadQueue builds queue, discards current PackageList
-func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool) (queue []PackageDownloadTask, downloadSize int64, err error) {
+func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
 	queue = make([]PackageDownloadTask, 0, repo.packageList.Len())
-	seen := make(map[string]struct{}, repo.packageList.Len())
+	seen := make(map[string]int, repo.packageList.Len())
 
 	err = repo.packageList.ForEach(func(p *Package) error {
-		list, err2 := p.DownloadList(packagePool)
+		if repo.packageRefs != nil && skipExistingPackages {
+			if repo.packageRefs.Has(p) {
+				// skip this package, but load checksums/files from package in DB
+				var prevP *Package
+				prevP, err = packageCollection.ByKey(p.Key(""))
+				if err != nil {
+					return err
+				}
+
+				p.UpdateFiles(prevP.Files())
+				return nil
+			}
+		}
+
+		list, err2 := p.DownloadList(packagePool, checksumStorage)
 		if err2 != nil {
 			return err2
 		}
-		p.files = nil
 
 		for _, task := range list {
-			key := task.RepoURI + "-" + task.DestinationPath
-			_, found := seen[key]
+			key := task.File.DownloadURL()
+			idx, found := seen[key]
 			if !found {
 				queue = append(queue, task)
-				downloadSize += task.Checksums.Size
-				seen[key] = struct{}{}
+				downloadSize += task.File.Checksums.Size
+				seen[key] = len(queue) - 1
+			} else {
+				// hook up the task to duplicate entry already on the list
+				queue[idx].Additional = append(queue[idx].Additional, task)
 			}
 		}
 
@@ -533,17 +540,39 @@ func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool) (queue
 		return
 	}
 
-	repo.tempPackageRefs = NewPackageRefListFromPackageList(repo.packageList)
-	// free up package list, we don't need it after this point
-	repo.packageList = nil
-
 	return
 }
 
 // FinalizeDownload swaps for final value of package refs
-func (repo *RemoteRepo) FinalizeDownload() {
+func (repo *RemoteRepo) FinalizeDownload(collectionFactory *CollectionFactory, progress aptly.Progress) error {
 	repo.LastDownloadDate = time.Now()
-	repo.packageRefs = repo.tempPackageRefs
+
+	if progress != nil {
+		progress.InitBar(int64(repo.packageList.Len()), true)
+	}
+
+	var i int
+
+	// update all the packages in collection
+	err := repo.packageList.ForEach(func(p *Package) error {
+		i++
+		if progress != nil {
+			progress.SetBar(i)
+		}
+		// download process might have updated checksums
+		p.UpdateFiles(p.Files())
+		return collectionFactory.PackageCollection().Update(p)
+	})
+
+	repo.packageRefs = NewPackageRefListFromPackageList(repo.packageList)
+
+	if progress != nil {
+		progress.ShutdownBar()
+	}
+
+	repo.packageList = nil
+
+	return err
 }
 
 // Encode does msgpack encoding of RemoteRepo

deb/remote_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 
 	. "gopkg.in/check.v1"
@@ -27,11 +28,11 @@ func (n *NullVerifier) InitKeyring() error {
 func (n *NullVerifier) AddKeyring(keyring string) {
 }
 
-func (n *NullVerifier) VerifyDetachedSignature(signature, cleartext io.Reader) error {
+func (n *NullVerifier) VerifyDetachedSignature(signature, cleartext io.Reader, hint bool) error {
 	return nil
 }
 
-func (n *NullVerifier) VerifyClearsigned(clearsigned io.Reader, hint bool) (*utils.GpgKeyInfo, error) {
+func (n *NullVerifier) VerifyClearsigned(clearsigned io.Reader, hint bool) (*pgp.KeyInfo, error) {
 	return nil, nil
 }
 
@@ -81,6 +82,7 @@ type RemoteRepoSuite struct {
 	db                database.Storage
 	collectionFactory *CollectionFactory
 	packagePool       aptly.PackagePool
+	cs                aptly.ChecksumStorage
 }
 
 var _ = Suite(&RemoteRepoSuite{})
@@ -90,9 +92,10 @@ func (s *RemoteRepoSuite) SetUpTest(c *C) {
 	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false)
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress()
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collectionFactory = NewCollectionFactory(s.db)
-	s.packagePool = files.NewPackagePool(c.MkDir())
+	s.packagePool = files.NewPackagePool(c.MkDir(), false)
+	s.cs = files.NewMockChecksumStorage()
 	s.SetUpPackages()
 	s.progress.Start()
 }
@@ -155,24 +158,30 @@ func (s *RemoteRepoSuite) TestReleaseURL(c *C) {
 	c.Assert(s.flat.ReleaseURL("Release").String(), Equals, "http://repos.express42.com/virool/precise/Release")
 }
 
-func (s *RemoteRepoSuite) TestBinaryURL(c *C) {
-	c.Assert(s.repo.BinaryURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/binary-amd64/Packages")
+func (s *RemoteRepoSuite) TestIndexesRootURL(c *C) {
+	c.Assert(s.repo.IndexesRootURL().String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/")
+
+	c.Assert(s.flat.IndexesRootURL().String(), Equals, "http://repos.express42.com/virool/precise/")
 }
 
-func (s *RemoteRepoSuite) TestUdebURL(c *C) {
-	c.Assert(s.repo.UdebURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/debian-installer/binary-amd64/Packages")
+func (s *RemoteRepoSuite) TestBinaryPath(c *C) {
+	c.Assert(s.repo.BinaryPath("main", "amd64"), Equals, "main/binary-amd64/Packages")
 }
 
-func (s *RemoteRepoSuite) TestSourcesURL(c *C) {
-	c.Assert(s.repo.SourcesURL("main").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources")
+func (s *RemoteRepoSuite) TestUdebPath(c *C) {
+	c.Assert(s.repo.UdebPath("main", "amd64"), Equals, "main/debian-installer/binary-amd64/Packages")
 }
 
-func (s *RemoteRepoSuite) TestFlatBinaryURL(c *C) {
-	c.Assert(s.flat.FlatBinaryURL().String(), Equals, "http://repos.express42.com/virool/precise/Packages")
+func (s *RemoteRepoSuite) TestSourcesPath(c *C) {
+	c.Assert(s.repo.SourcesPath("main"), Equals, "main/source/Sources")
 }
 
-func (s *RemoteRepoSuite) TestFlatSourcesURL(c *C) {
-	c.Assert(s.flat.FlatSourcesURL().String(), Equals, "http://repos.express42.com/virool/precise/Sources")
+func (s *RemoteRepoSuite) TestFlatBinaryPath(c *C) {
+	c.Assert(s.flat.FlatBinaryPath(), Equals, "Packages")
+}
+
+func (s *RemoteRepoSuite) TestFlatSourcesPath(c *C) {
+	c.Assert(s.flat.FlatSourcesPath(), Equals, "Sources")
 }
 
 func (s *RemoteRepoSuite) TestPackageURL(c *C) {
@@ -266,18 +275,62 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(3))
 	c.Check(queue, HasLen, 1)
-	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+	c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
 
-	s.repo.FinalizeDownload()
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.repo.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
 }
 
 func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
@@ -298,13 +351,14 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(15))
 	c.Check(queue, HasLen, 4)
 
 	q := make([]string, 4)
 	for i := range q {
-		q[i] = queue[i].RepoURI
+		q[i] = queue[i].File.DownloadURL()
 	}
 	sort.Strings(q)
 	c.Check(q[3], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
@@ -312,7 +366,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	c.Check(q[2], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz")
 	c.Check(q[0], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz")
 
-	s.repo.FinalizeDownload()
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.repo.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
@@ -323,6 +377,56 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
 }
 
 func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
@@ -340,18 +444,64 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(3))
 	c.Check(queue, HasLen, 1)
-	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+	c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
 
-	s.flat.FinalizeDownload()
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.flat.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.flat.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.flat.packageRefs, NotNil)
 }
 
 func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
@@ -375,13 +525,14 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(15))
 	c.Check(queue, HasLen, 4)
 
 	q := make([]string, 4)
 	for i := range q {
-		q[i] = queue[i].RepoURI
+		q[i] = queue[i].File.DownloadURL()
 	}
 	sort.Strings(q)
 	c.Check(q[3], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
@@ -389,7 +540,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Check(q[2], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz")
 	c.Check(q[0], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz")
 
-	s.flat.FinalizeDownload()
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.flat.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
@@ -401,6 +552,58 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.flat.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.flat.packageRefs, NotNil)
 }
 
 type RemoteRepoCollectionSuite struct {
@@ -412,7 +615,7 @@ type RemoteRepoCollectionSuite struct {
 var _ = Suite(&RemoteRepoCollectionSuite{})
 
 func (s *RemoteRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collection = NewRemoteRepoCollection(s.db)
 	s.SetUpPackages()
 }
@@ -422,14 +625,14 @@ func (s *RemoteRepoCollectionSuite) TearDownTest(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
-	r, err := s.collection.ByName("yandex")
+	_, err := s.collection.ByName("yandex")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
-	r, err = s.collection.ByName("yandex")
+	r, err := s.collection.ByName("yandex")
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 
@@ -440,13 +643,13 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
-	r, err := s.collection.ByUUID("some-uuid")
+	_, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 
-	r, err = s.collection.ByUUID(repo.UUID)
+	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 }

deb/snapshot.go

@@ -31,6 +31,10 @@ type Snapshot struct {
 	// Description of how snapshot was created
 	Description string
 
+	Origin               string
+	NotAutomatic         string
+	ButAutomaticUpgrades string
+
 	packageRefs *PackageRefList
 }
 
@@ -41,31 +45,36 @@ func NewSnapshotFromRepository(name string, repo *RemoteRepo) (*Snapshot, error)
 	}
 
 	return &Snapshot{
-		UUID:        uuid.New(),
-		Name:        name,
-		CreatedAt:   time.Now(),
-		SourceKind:  "repo",
-		SourceIDs:   []string{repo.UUID},
-		Description: fmt.Sprintf("Snapshot from mirror %s", repo),
-		packageRefs: repo.packageRefs,
+		UUID:                 uuid.New(),
+		Name:                 name,
+		CreatedAt:            time.Now(),
+		SourceKind:           SourceRemoteRepo,
+		SourceIDs:            []string{repo.UUID},
+		Description:          fmt.Sprintf("Snapshot from mirror %s", repo),
+		Origin:               repo.Meta["Origin"],
+		NotAutomatic:         repo.Meta["NotAutomatic"],
+		ButAutomaticUpgrades: repo.Meta["ButAutomaticUpgrades"],
+		packageRefs:          repo.packageRefs,
 	}, nil
 }
 
 // NewSnapshotFromLocalRepo creates snapshot from current state of local repository
 func NewSnapshotFromLocalRepo(name string, repo *LocalRepo) (*Snapshot, error) {
-	if repo.packageRefs == nil {
-		return nil, errors.New("local repo doesn't have packages")
-	}
-
-	return &Snapshot{
+	snap := &Snapshot{
 		UUID:        uuid.New(),
 		Name:        name,
 		CreatedAt:   time.Now(),
-		SourceKind:  "local",
+		SourceKind:  SourceLocalRepo,
 		SourceIDs:   []string{repo.UUID},
 		Description: fmt.Sprintf("Snapshot from local repo %s", repo),
 		packageRefs: repo.packageRefs,
-	}, nil
+	}
+
+	if snap.packageRefs == nil {
+		snap.packageRefs = NewPackageRefList()
+	}
+
+	return snap, nil
 }
 
 // NewSnapshotFromPackageList creates snapshot from PackageList
@@ -255,7 +264,7 @@ func (collection *SnapshotCollection) ByRemoteRepoSource(repo *RemoteRepo) []*Sn
 	var result []*Snapshot
 
 	for _, s := range collection.list {
-		if s.SourceKind == "repo" && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
+		if s.SourceKind == SourceRemoteRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
 			result = append(result, s)
 		}
 	}
@@ -267,7 +276,7 @@ func (collection *SnapshotCollection) ByLocalRepoSource(repo *LocalRepo) []*Snap
 	var result []*Snapshot
 
 	for _, s := range collection.list {
-		if s.SourceKind == "local" && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
+		if s.SourceKind == SourceLocalRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
 			result = append(result, s)
 		}
 	}

deb/snapshot_test.go

@@ -26,7 +26,7 @@ func (s *SnapshotSuite) TestNewSnapshotFromRepository(c *C) {
 	c.Check(snapshot.Name, Equals, "snap1")
 	c.Check(snapshot.NumPackages(), Equals, 3)
 	c.Check(snapshot.RefList().Len(), Equals, 3)
-	c.Check(snapshot.SourceKind, Equals, "repo")
+	c.Check(snapshot.SourceKind, Equals, SourceRemoteRepo)
 	c.Check(snapshot.SourceIDs, DeepEquals, []string{s.repo.UUID})
 
 	s.repo.packageRefs = nil
@@ -37,11 +37,17 @@ func (s *SnapshotSuite) TestNewSnapshotFromRepository(c *C) {
 func (s *SnapshotSuite) TestNewSnapshotFromLocalRepo(c *C) {
 	localRepo := NewLocalRepo("lala", "hoorah!")
 
-	_, err := NewSnapshotFromLocalRepo("snap2", localRepo)
-	c.Check(err, ErrorMatches, "local repo doesn't have packages")
+	snapshot, err := NewSnapshotFromLocalRepo("snap2", localRepo)
+	c.Assert(err, IsNil)
+	c.Check(snapshot.Name, Equals, "snap2")
+	c.Check(snapshot.NumPackages(), Equals, 0)
+	c.Check(snapshot.RefList().Len(), Equals, 0)
+	c.Check(snapshot.SourceKind, Equals, "local")
+	c.Check(snapshot.SourceIDs, DeepEquals, []string{localRepo.UUID})
 
 	localRepo.UpdateRefList(s.reflist)
-	snapshot, _ := NewSnapshotFromLocalRepo("snap1", localRepo)
+	snapshot, err = NewSnapshotFromLocalRepo("snap1", localRepo)
+	c.Assert(err, IsNil)
 	c.Check(snapshot.Name, Equals, "snap1")
 	c.Check(snapshot.NumPackages(), Equals, 3)
 	c.Check(snapshot.RefList().Len(), Equals, 3)
@@ -106,7 +112,7 @@ type SnapshotCollectionSuite struct {
 var _ = Suite(&SnapshotCollectionSuite{})
 
 func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collection = NewSnapshotCollection(s.db)
 	s.SetUpPackages()
 
@@ -132,7 +138,7 @@ func (s *SnapshotCollectionSuite) TearDownTest(c *C) {
 }
 
 func (s *SnapshotCollectionSuite) TestAddByNameByUUID(c *C) {
-	snapshot, err := s.collection.ByName("snap1")
+	_, err := s.collection.ByName("snap1")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	c.Assert(s.collection.Add(s.snapshot1), IsNil)
@@ -140,7 +146,7 @@ func (s *SnapshotCollectionSuite) TestAddByNameByUUID(c *C) {
 
 	c.Assert(s.collection.Add(s.snapshot2), IsNil)
 
-	snapshot, err = s.collection.ByName("snap1")
+	snapshot, err := s.collection.ByName("snap1")
 	c.Assert(err, IsNil)
 	c.Assert(snapshot.String(), Equals, s.snapshot1.String())
 

deb/uploaders.go

@@ -6,6 +6,7 @@ import (
 	"os"
 
 	"github.com/DisposaBoy/JsonConfigReader"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -85,7 +86,7 @@ func (u *Uploaders) IsAllowed(changes *Changes) error {
 			deny := u.ExpandGroups(rule.Deny)
 			for _, key := range changes.SignatureKeys {
 				for _, item := range deny {
-					if item == "*" || key.Matches(utils.GpgKey(item)) {
+					if item == "*" || key.Matches(pgp.Key(item)) {
 						return fmt.Errorf("denied according to rule: %s", rule)
 					}
 				}
@@ -94,7 +95,7 @@ func (u *Uploaders) IsAllowed(changes *Changes) error {
 			allow := u.ExpandGroups(rule.Allow)
 			for _, key := range changes.SignatureKeys {
 				for _, item := range allow {
-					if item == "*" || key.Matches(utils.GpgKey(item)) {
+					if item == "*" || key.Matches(pgp.Key(item)) {
 						return nil
 					}
 				}

deb/uploaders_test.go

@@ -1,7 +1,7 @@
 package deb
 
 import (
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/pgp"
 	. "gopkg.in/check.v1"
 )
 
@@ -58,24 +58,24 @@ func (s *UploadersSuite) TestIsAllowed(c *C) {
 	}
 
 	// no keys - not allowed
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{}, Stanza: Stanza{"Source": "calamares"}}), ErrorMatches, "denied as no rule matches")
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{}, Stanza: Stanza{"Source": "calamares"}}), ErrorMatches, "denied as no rule matches")
 
 	// no rule - not allowed
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"37E1C17570096AD1", "EC4B033C70096AD1"}, Stanza: Stanza{"Source": "unknown-calamares"}}), ErrorMatches, "denied as no rule matches")
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"37E1C17570096AD1", "EC4B033C70096AD1"}, Stanza: Stanza{"Source": "unknown-calamares"}}), ErrorMatches, "denied as no rule matches")
 
 	// first rule: allow anyone do stuff with calamares
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "calamares"}}), IsNil)
 
 	// second rule: nobody is allowed to do stuff with never-calamares
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "never-calamares"}}),
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "never-calamares"}}),
 		ErrorMatches, "denied according to rule: {\"condition\":\"\",\"allow\":null,\"deny\":\\[\"\\*\"\\]}")
 
 	// third rule: anyone from the group or explicit key
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"45678901", "12345678"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"37E1C17570096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"70096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"45678901", "12345678"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"37E1C17570096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"70096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
 
 	// fourth rule: some are not allowed
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "45678901"}, Stanza: Stanza{"Source": "some-calamares"}}),
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"ABCD1234", "45678901"}, Stanza: Stanza{"Source": "some-calamares"}}),
 		ErrorMatches, "denied according to rule: {\"condition\":\"\",\"allow\":null,\"deny\":\\[\"45678901\",\"12345678\"\\]}")
 }