New upstream version 1.5.0+ds1

deb/changes.go

@@ -195,11 +195,6 @@ func (c *Changes) PackageQuery() PackageQuery {
 			}
 		}
 
-		ddebQuery = &AndQuery{
-			L: &FieldQuery{Field: "Source", Relation: VersionEqual, Value: c.Source},
-			R: ddebQuery,
-		}
-
 		binaryQuery = &OrQuery{
 			L: binaryQuery,
 			R: ddebQuery,
@@ -293,14 +288,9 @@ func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (cha
 
 // ImportChangesFiles imports referenced files in changes files into local repository
 func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles bool,
-	verifier pgp.Verifier, repoTemplateString string, progress aptly.Progress, localRepoCollection *LocalRepoCollection, packageCollection *PackageCollection,
-	pool aptly.PackagePool, checksumStorage aptly.ChecksumStorage, uploaders *Uploaders, parseQuery parseQuery) (processedFiles []string, failedFiles []string, err error) {
+	verifier pgp.Verifier, repoTemplate *template.Template, progress aptly.Progress, localRepoCollection *LocalRepoCollection, packageCollection *PackageCollection,
+	pool aptly.PackagePool, checksumStorageProvider aptly.ChecksumStorageProvider, uploaders *Uploaders, parseQuery parseQuery) (processedFiles []string, failedFiles []string, err error) {
 
-	var repoTemplate *template.Template
-	repoTemplate, err = template.New("repo").Parse(repoTemplateString)
-	if err != nil {
-		return nil, nil, fmt.Errorf("error parsing -repo template: %s", err)
-	}
 	for _, path := range changesFiles {
 		var changes *Changes
 
@@ -384,7 +374,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
 		var processedFiles2, failedFiles2 []string
 
 		processedFiles2, failedFiles2, err = ImportPackageFiles(list, packageFiles, forceReplace, verifier, pool,
-			packageCollection, reporter, restriction, checksumStorage)
+			packageCollection, reporter, restriction, checksumStorageProvider)
 
 		if err != nil {
 			return nil, nil, fmt.Errorf("unable to import package files: %s", err)

deb/changes_test.go

@@ -3,10 +3,12 @@ package deb
 import (
 	"os"
 	"path/filepath"
+	"text/template"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/console"
 	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/utils"
 
@@ -37,7 +39,7 @@ func (s *ChangesSuite) SetUpTest(c *C) {
 	err := utils.CopyFile("testdata/changes/calamares.changes", s.Path)
 	c.Assert(err, IsNil)
 
-	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.localRepoCollection = NewLocalRepoCollection(s.db)
 	s.packageCollection = NewPackageCollection(s.db)
 
@@ -122,13 +124,30 @@ func (s *ChangesSuite) TestImportChangesFiles(c *C) {
 	processedFiles, failedFiles, err := ImportChangesFiles(
 		append(changesFiles, "testdata/changes/notexistent.changes"),
 		s.Reporter, true, true, false, false, &NullVerifier{},
-		"test", s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, s.checksumStorage,
+		template.Must(template.New("test").Parse("test")), s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, func(database.ReaderWriter) aptly.ChecksumStorage { return s.checksumStorage },
 		nil, nil)
 	c.Assert(err, IsNil)
 	c.Check(failedFiles, DeepEquals, append(expectedFailedFiles, "testdata/changes/notexistent.changes"))
 	c.Check(processedFiles, DeepEquals, expectedProcessedFiles)
 }
 
+func (s *ChangesSuite) TestImportDbgsymWithVersionedSourceField(c *C) {
+	repo := NewLocalRepo("test", "Test Comment")
+	c.Assert(s.localRepoCollection.Add(repo), IsNil)
+
+	changesFiles, failedFiles := CollectChangesFiles(
+		[]string{"testdata/dbgsym-with-source-version"}, s.Reporter)
+	c.Check(changesFiles, HasLen, 1)
+	c.Check(failedFiles, HasLen, 0)
+
+	_, failedFiles, err := ImportChangesFiles(
+		changesFiles, s.Reporter, true, true, false, true, &NullVerifier{},
+		template.Must(template.New("test").Parse("test")), s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, func(database.ReaderWriter) aptly.ChecksumStorage { return s.checksumStorage },
+		nil, nil)
+	c.Assert(err, IsNil)
+	c.Check(failedFiles, IsNil)
+}
+
 func (s *ChangesSuite) TestPrepare(c *C) {
 	changes, err := NewChanges("testdata/changes/hardlink_0.2.1_amd64.changes")
 	c.Assert(err, IsNil)
@@ -148,5 +167,5 @@ func (s *ChangesSuite) TestPackageQuery(c *C) {
 
 	q := changes.PackageQuery()
 	c.Check(q.String(), Equals,
-		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), (((Name (= calamares-dbg)) | (Name (= calamares))) | ((Source (= calamares)), ((Name (= calamares-dbg-dbgsym)) | (Name (= calamares-dbgsym)))))))")
+		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), (((Name (= calamares-dbg)) | (Name (= calamares))) | ((Name (= calamares-dbg-dbgsym)) | (Name (= calamares-dbgsym))))))")
 }

deb/checksum_collection.go

@@ -11,12 +11,12 @@ import (
 
 // ChecksumCollection does management of ChecksumInfo in DB
 type ChecksumCollection struct {
-	db          database.Storage
+	db          database.ReaderWriter
 	codecHandle *codec.MsgpackHandle
 }
 
 // NewChecksumCollection creates new ChecksumCollection and binds it to database
-func NewChecksumCollection(db database.Storage) *ChecksumCollection {
+func NewChecksumCollection(db database.ReaderWriter) *ChecksumCollection {
 	return &ChecksumCollection{
 		db:          db,
 		codecHandle: &codec.MsgpackHandle{},

deb/checksum_collection_test.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
 	"github.com/aptly-dev/aptly/utils"
 
 	. "gopkg.in/check.v1"
@@ -22,7 +23,7 @@ func (s *ChecksumCollectionSuite) SetUpTest(c *C) {
 		SHA1:   "da39a3ee5e6b4b0d3255bfef95601890afd80709",
 		SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
 	}
-	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.collection = NewChecksumCollection(s.db)
 }
 

deb/collections.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"sync"
 
+	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
 )
 
@@ -91,10 +92,14 @@ func (factory *CollectionFactory) PublishedRepoCollection() *PublishedRepoCollec
 }
 
 // ChecksumCollection returns (or creates) new ChecksumCollection
-func (factory *CollectionFactory) ChecksumCollection() *ChecksumCollection {
+func (factory *CollectionFactory) ChecksumCollection(db database.ReaderWriter) aptly.ChecksumStorage {
 	factory.Lock()
 	defer factory.Unlock()
 
+	if db != nil {
+		return NewChecksumCollection(db)
+	}
+
 	if factory.checksums == nil {
 		factory.checksums = NewChecksumCollection(factory.db)
 	}

deb/contents.go

@@ -25,11 +25,11 @@ func NewContentsIndex(db database.Storage) *ContentsIndex {
 }
 
 // Push adds package to contents index, calculating package contents as required
-func (index *ContentsIndex) Push(qualifiedName []byte, contents []string) error {
+func (index *ContentsIndex) Push(qualifiedName []byte, contents []string, dbw database.Writer) error {
 	for _, path := range contents {
 		// for performance reasons we only write to leveldb during push.
 		// merging of qualified names per path will be done in WriteTo
-		err := index.db.Put(append(append(append(index.prefix, []byte(path)...), byte(0)), qualifiedName...), nil)
+		err := dbw.Put(append(append(append(index.prefix, []byte(path)...), byte(0)), qualifiedName...), nil)
 		if err != nil {
 			return err
 		}

deb/deb.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/kjk/lzma"
+	"github.com/klauspost/compress/zstd"
 	"github.com/smira/go-xz"
 )
 
@@ -74,6 +75,13 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 				}
 				defer unxz.Close()
 				tarInput = unxz
+			case "control.tar.zst":
+				unzstd, err := zstd.NewReader(bufReader)
+				if err != nil {
+					return nil, errors.Wrapf(err, "unable to unzstd %s from %s", header.Name, packageFile)
+				}
+				defer unzstd.Close()
+				tarInput = unzstd
 			default:
 				return nil, fmt.Errorf("unsupported tar compression in %s: %s", packageFile, header.Name)
 			}
@@ -189,6 +197,13 @@ func GetContentsFromDeb(file io.Reader, packageFile string) ([]string, error) {
 				unlzma := lzma.NewReader(bufReader)
 				defer unlzma.Close()
 				tarInput = unlzma
+			case "data.tar.zst":
+				unzstd, err := zstd.NewReader(bufReader)
+				if err != nil {
+					return nil, errors.Wrapf(err, "unable to unzstd %s from %s", header.Name, packageFile)
+				}
+				defer unzstd.Close()
+				tarInput = unzstd
 			default:
 				return nil, fmt.Errorf("unsupported tar compression in %s: %s", packageFile, header.Name)
 			}

deb/deb_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 type DebSuite struct {
-	debFile, debFile2, debFileWithXzControl, dscFile, dscFileNoSign string
+	debFile, debFile2, debFileWithXzControl, debFileWithZstdControl, dscFile, dscFileNoSign string
 }
 
 var _ = Suite(&DebSuite{})
@@ -21,6 +21,7 @@ func (s *DebSuite) SetUpSuite(c *C) {
 	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
 	s.debFile2 = filepath.Join(filepath.Dir(_File), "../system/changes/hardlink_0.2.1_amd64.deb")
 	s.debFileWithXzControl = filepath.Join(filepath.Dir(_File), "../system/changes/libqt5concurrent5-dbgsym_5.9.1+dfsg-2+18.04+bionic+build4_amd64.ddeb")
+	s.debFileWithZstdControl = filepath.Join(filepath.Dir(_File), "../system/changes/libqt5concurrent5-dbgsym_5.15.2+dfsg-12_amd64.ddeb")
 	s.dscFile = filepath.Join(filepath.Dir(_File), "../system/files/pyspi_0.6.1-1.3.dsc")
 	s.dscFileNoSign = filepath.Join(filepath.Dir(_File), "../system/files/pyspi-0.6.1-1.3.stripped.dsc")
 }
@@ -47,6 +48,14 @@ func (s *DebSuite) TestGetControlFileFromDebWithXzControl(c *C) {
 	c.Check(st["Package"], Equals, "libqt5concurrent5-dbgsym")
 }
 
+func (s *DebSuite) TestGetControlFileFromDebWithZstdControl(c *C) {
+	// Has control.tar.zstd archive inside.
+	st, err := GetControlFileFromDeb(s.debFileWithZstdControl)
+	c.Check(err, IsNil)
+	c.Check(st["Version"], Equals, "5.15.2+dfsg-12")
+	c.Check(st["Package"], Equals, "libqt5concurrent5-dbgsym")
+}
+
 func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
 	verifier := &pgp.GoVerifier{}
 

deb/import.go

@@ -66,11 +66,19 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 // ImportPackageFiles imports files into local repository
 func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier pgp.Verifier,
 	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter, restriction PackageQuery,
-	checksumStorage aptly.ChecksumStorage) (processedFiles []string, failedFiles []string, err error) {
+	checksumStorageProvider aptly.ChecksumStorageProvider) (processedFiles []string, failedFiles []string, err error) {
 	if forceReplace {
 		list.PrepareIndex()
 	}
 
+	transaction, err := collection.db.OpenTransaction()
+	if err != nil {
+		return nil, nil, err
+	}
+	defer transaction.Discard()
+
+	checksumStorage := checksumStorageProvider(transaction)
+
 	for _, file := range packageFiles {
 		var (
 			stanza Stanza
@@ -193,7 +201,7 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
-		err = collection.Update(p)
+		err = collection.UpdateInTransaction(p, transaction)
 		if err != nil {
 			reporter.Warning("Unable to save package %s: %s", p, err)
 			failedFiles = append(failedFiles, file)
@@ -219,6 +227,6 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 		processedFiles = append(processedFiles, candidateProcessedFiles...)
 	}
 
-	err = nil
+	err = transaction.Commit()
 	return
 }

deb/index_files.go

@@ -22,6 +22,7 @@ type indexFiles struct {
 	suffix           string
 	indexes          map[string]*indexFile
 	acquireByHash    bool
+	skipBz2          bool
 }
 
 type indexFile struct {
@@ -68,7 +69,7 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 	}
 
 	if file.compressable {
-		err = utils.CompressFile(file.tempFile, file.onlyGzip)
+		err = utils.CompressFile(file.tempFile, file.onlyGzip || file.parent.skipBz2)
 		if err != nil {
 			file.tempFile.Close()
 			return fmt.Errorf("unable to compress index file: %s", err)
@@ -80,11 +81,15 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 	exts := []string{""}
 	cksumExts := exts
 	if file.compressable {
-		exts = append(exts, ".gz", ".bz2")
-		cksumExts = exts
 		if file.onlyGzip {
 			exts = []string{".gz"}
 			cksumExts = []string{"", ".gz"}
+		} else {
+			exts = append(exts, ".gz")
+			if !file.parent.skipBz2 {
+				exts = append(exts, ".bz2")
+			}
+			cksumExts = exts
 		}
 	}
 
@@ -229,7 +234,7 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er
 	return nil
 }
 
-func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool) *indexFiles {
+func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool, skipBz2 bool) *indexFiles {
 	return &indexFiles{
 		publishedStorage: publishedStorage,
 		basePath:         basePath,
@@ -239,6 +244,7 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 		suffix:           suffix,
 		indexes:          make(map[string]*indexFile),
 		acquireByHash:    acquireByHash,
+		skipBz2:          skipBz2,
 	}
 }
 
@@ -391,7 +397,7 @@ func (files *indexFiles) ReleaseFile() *indexFile {
 
 func (files *indexFiles) FinalizeAll(progress aptly.Progress, signer pgp.Signer) (err error) {
 	if progress != nil {
-		progress.InitBar(int64(len(files.indexes)), false)
+		progress.InitBar(int64(len(files.indexes)), false, aptly.BarPublishFinalizeIndexes)
 		defer progress.ShutdownBar()
 	}
 

deb/list.go

@@ -99,7 +99,7 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 	result := NewPackageListWithDuplicates(false, reflist.Len())
 
 	if progress != nil {
-		progress.InitBar(int64(reflist.Len()), false)
+		progress.InitBar(int64(reflist.Len()), false, aptly.BarGeneralBuildPackageList)
 	}
 
 	err := reflist.ForEach(func(key []byte) error {
@@ -266,6 +266,19 @@ func (l *PackageList) Strings() []string {
 	return result
 }
 
+// FullNames builds a list of package {name}_{version}_{arch}
+func (l *PackageList) FullNames() []string {
+	result := make([]string, l.Len())
+	i := 0
+
+	for _, p := range l.packages {
+		result[i] = p.GetFullName()
+		i++
+	}
+
+	return result
+}
+
 // depSliceDeduplicate removes dups in slice of Dependencies
 func depSliceDeduplicate(s []Dependency) []Dependency {
 	l := len(s)
@@ -301,7 +314,7 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 	missing := make([]Dependency, 0, 128)
 
 	if progress != nil {
-		progress.InitBar(int64(l.Len())*int64(len(architectures)), false)
+		progress.InitBar(int64(l.Len())*int64(len(architectures)), false, aptly.BarGeneralVerifyDependencies)
 	}
 
 	for _, arch := range architectures {

deb/local.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"sync"
 
 	"github.com/aptly-dev/aptly/database"
 	"github.com/pborman/uuid"
@@ -15,7 +14,7 @@ import (
 // LocalRepo is a collection of packages created locally
 type LocalRepo struct {
 	// Permanent internal ID
-	UUID string `json:"-"`
+	UUID string `codec:"UUID" json:"-"`
 	// User-assigned name
 	Name string
 	// Comment
@@ -25,7 +24,7 @@ type LocalRepo struct {
 	// DefaultComponent
 	DefaultComponent string `codec:",omitempty"`
 	// Uploaders configuration
-	Uploaders *Uploaders `code:",omitempty" json:"-"`
+	Uploaders *Uploaders `codec:"Uploaders,omitempty" json:"-"`
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 }
@@ -93,7 +92,6 @@ func (repo *LocalRepo) RefKey() []byte {
 
 // LocalRepoCollection does listing, updating/adding/deleting of LocalRepos
 type LocalRepoCollection struct {
-	*sync.RWMutex
 	db    database.Storage
 	cache map[string]*LocalRepo
 }
@@ -101,9 +99,8 @@ type LocalRepoCollection struct {
 // NewLocalRepoCollection loads LocalRepos from DB and makes up collection
 func NewLocalRepoCollection(db database.Storage) *LocalRepoCollection {
 	return &LocalRepoCollection{
-		RWMutex: &sync.RWMutex{},
-		db:      db,
-		cache:   make(map[string]*LocalRepo),
+		db:    db,
+		cache: make(map[string]*LocalRepo),
 	}
 }
 
@@ -161,17 +158,12 @@ func (collection *LocalRepoCollection) Add(repo *LocalRepo) error {
 
 // Update stores updated information about repo in DB
 func (collection *LocalRepoCollection) Update(repo *LocalRepo) error {
-	err := collection.db.Put(repo.Key(), repo.Encode())
-	if err != nil {
-		return err
-	}
+	batch := collection.db.CreateBatch()
+	batch.Put(repo.Key(), repo.Encode())
 	if repo.packageRefs != nil {
-		err = collection.db.Put(repo.RefKey(), repo.packageRefs.Encode())
-		if err != nil {
-			return err
-		}
+		batch.Put(repo.RefKey(), repo.packageRefs.Encode())
 	}
-	return nil
+	return batch.Write()
 }
 
 // LoadComplete loads additional information for local repo
@@ -245,16 +237,17 @@ func (collection *LocalRepoCollection) Len() int {
 
 // Drop removes remote repo from collection
 func (collection *LocalRepoCollection) Drop(repo *LocalRepo) error {
-	if _, err := collection.db.Get(repo.Key()); err == database.ErrNotFound {
-		panic("local repo not found!")
-	}
+	if _, err := collection.db.Get(repo.Key()); err != nil {
+		if err == database.ErrNotFound {
+			return errors.New("local repo not found")
+		}
 
-	delete(collection.cache, repo.UUID)
-
-	err := collection.db.Delete(repo.Key())
-	if err != nil {
 		return err
 	}
+	delete(collection.cache, repo.UUID)
 
-	return collection.db.Delete(repo.RefKey())
+	batch := collection.db.CreateBatch()
+	batch.Delete(repo.Key())
+	batch.Delete(repo.RefKey())
+	return batch.Write()
 }

deb/local_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 
 	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
 
 	. "gopkg.in/check.v1"
 )
@@ -18,7 +19,7 @@ type LocalRepoSuite struct {
 var _ = Suite(&LocalRepoSuite{})
 
 func (s *LocalRepoSuite) SetUpTest(c *C) {
-	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.list = NewPackageList()
 	s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
 	s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
@@ -83,7 +84,7 @@ type LocalRepoCollectionSuite struct {
 var _ = Suite(&LocalRepoCollectionSuite{})
 
 func (s *LocalRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.collection = NewLocalRepoCollection(s.db)
 
 	s.list = NewPackageList()
@@ -198,5 +199,15 @@ func (s *LocalRepoCollectionSuite) TestDrop(c *C) {
 	r2, _ := collection.ByName("local2")
 	c.Check(r2.String(), Equals, repo2.String())
 
-	c.Check(func() { s.collection.Drop(repo1) }, Panics, "local repo not found!")
+	c.Check(s.collection.Drop(repo1), ErrorMatches, "local repo not found")
+}
+
+func (s *LocalRepoCollectionSuite) TestDropNonExisting(c *C) {
+	repo := NewLocalRepo("local3", "Comment 3")
+
+	_, err := s.collection.ByUUID(repo.UUID)
+	c.Check(err, ErrorMatches, "local repo .* not found")
+
+	err = s.collection.Drop(repo)
+	c.Check(s.collection.Drop(repo), ErrorMatches, "local repo not found")
 }

deb/package.go

@@ -52,10 +52,10 @@ const (
 	PackageTypeInstaller = "installer"
 )
 
-// Special arhictectures
+// Special architectures
 const (
 	ArchitectureAll    = "all"
-	ArhictectureAny    = "any"
+	ArchitectureAny    = "any"
 	ArchitectureSource = "source"
 )
 
@@ -361,6 +361,11 @@ func (p *Package) GetName() string {
 	return p.Name
 }
 
+// GetFullName returns the package full name
+func (p *Package) GetFullName() string {
+	return strings.Join([]string{p.Name, p.Version, p.Architecture}, "_")
+}
+
 // GetVersion returns package version
 func (p *Package) GetVersion() string {
 	return p.Version

deb/package_collection.go

@@ -201,8 +201,23 @@ func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.
 	return contents
 }
 
-// Update adds or updates information about package in DB checking for conficts first
+// Update adds or updates information about package in DB
 func (collection *PackageCollection) Update(p *Package) error {
+	transaction, err := collection.db.OpenTransaction()
+	if err != nil {
+		return err
+	}
+	defer transaction.Discard()
+
+	if err = collection.UpdateInTransaction(p, transaction); err != nil {
+		return err
+	}
+
+	return transaction.Commit()
+}
+
+// UpdateInTransaction updates/creates package info in the context of the outer transaction
+func (collection *PackageCollection) UpdateInTransaction(p *Package, transaction database.Transaction) error {
 	var encodeBuffer bytes.Buffer
 
 	encoder := codec.NewEncoder(&encodeBuffer, collection.codecHandle)
@@ -210,12 +225,11 @@ func (collection *PackageCollection) Update(p *Package) error {
 	encodeBuffer.Reset()
 	encodeBuffer.WriteByte(0xc1)
 	encodeBuffer.WriteByte(0x1)
-	err := encoder.Encode(p)
-	if err != nil {
+	if err := encoder.Encode(p); err != nil {
 		return err
 	}
 
-	err = collection.db.Put(p.Key(""), encodeBuffer.Bytes())
+	err := transaction.Put(p.Key(""), encodeBuffer.Bytes())
 	if err != nil {
 		return err
 	}
@@ -228,7 +242,7 @@ func (collection *PackageCollection) Update(p *Package) error {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xF"), encodeBuffer.Bytes())
+		err = transaction.Put(p.Key("xF"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
@@ -241,7 +255,7 @@ func (collection *PackageCollection) Update(p *Package) error {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xD"), encodeBuffer.Bytes())
+		err = transaction.Put(p.Key("xD"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
@@ -256,7 +270,7 @@ func (collection *PackageCollection) Update(p *Package) error {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xE"), encodeBuffer.Bytes())
+		err = transaction.Put(p.Key("xE"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
@@ -265,7 +279,6 @@ func (collection *PackageCollection) Update(p *Package) error {
 	}
 
 	p.collection = collection
-
 	return nil
 }
 
@@ -275,9 +288,9 @@ func (collection *PackageCollection) AllPackageRefs() *PackageRefList {
 }
 
 // DeleteByKey deletes package in DB by key
-func (collection *PackageCollection) DeleteByKey(key []byte) error {
+func (collection *PackageCollection) DeleteByKey(key []byte, dbw database.Writer) error {
 	for _, key := range [][]byte{key, append([]byte("xF"), key...), append([]byte("xD"), key...), append([]byte("xE"), key...)} {
-		err := collection.db.Delete(key)
+		err := dbw.Delete(key)
 		if err != nil {
 			return err
 		}

deb/package_collection_test.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
 	"github.com/aptly-dev/aptly/utils"
 
 	. "gopkg.in/check.v1"
@@ -17,7 +18,7 @@ var _ = Suite(&PackageCollectionSuite{})
 
 func (s *PackageCollectionSuite) SetUpTest(c *C) {
 	s.p = NewPackageFromControlFile(packageStanza.Copy())
-	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.collection = NewPackageCollection(s.db)
 }
 
@@ -113,7 +114,7 @@ func (s *PackageCollectionSuite) TestDeleteByKey(c *C) {
 	_, err = s.db.Get(s.p.Key("xF"))
 	c.Check(err, IsNil)
 
-	err = s.collection.DeleteByKey(s.p.Key(""))
+	err = s.collection.DeleteByKey(s.p.Key(""), s.db)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByKey(s.p.Key(""))

deb/publish.go

@@ -11,7 +11,6 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/pborman/uuid"
@@ -44,6 +43,7 @@ type PublishedRepo struct {
 	NotAutomatic         string
 	ButAutomaticUpgrades string
 	Label                string
+	Suite                string
 	// Architectures is a list of all architectures published
 	Architectures []string
 	// SourceKind is "local"/"repo"
@@ -62,6 +62,9 @@ type PublishedRepo struct {
 	// Skip contents generation
 	SkipContents bool
 
+	// Skip bz2 compression for index files
+	SkipBz2 bool
+
 	// True if repo is being re-published
 	rePublishing bool
 
@@ -281,7 +284,7 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 	return result, nil
 }
 
-// MarshalJSON requires object to be "loeaded completely"
+// MarshalJSON requires object to be "loaded completely"
 func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
 	type sourceInfo struct {
 		Component, Name string
@@ -308,9 +311,11 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
 		"Distribution":         p.Distribution,
 		"Label":                p.Label,
 		"Origin":               p.Origin,
+		"Suite":                p.Suite,
 		"NotAutomatic":         p.NotAutomatic,
 		"ButAutomaticUpgrades": p.ButAutomaticUpgrades,
 		"Prefix":               p.Prefix,
+		"Path":                 p.GetPath(),
 		"SourceKind":           p.SourceKind,
 		"Sources":              sources,
 		"Storage":              p.Storage,
@@ -357,6 +362,10 @@ func (p *PublishedRepo) String() string {
 		extras = append(extras, fmt.Sprintf("label: %s", p.Label))
 	}
 
+	if p.Suite != "" {
+		extras = append(extras, fmt.Sprintf("suite: %s", p.Suite))
+	}
+
 	extra = strings.Join(extras, ", ")
 
 	if extra != "" {
@@ -485,6 +494,25 @@ func (p *PublishedRepo) GetLabel() string {
 	return p.Label
 }
 
+// GetPath returns the unique name of the repo
+func (p *PublishedRepo) GetPath() string {
+	prefix := p.StoragePrefix()
+
+	if prefix == "" {
+		return p.Distribution
+	}
+
+	return fmt.Sprintf("%s/%s", prefix, p.Distribution)
+}
+
+// GetSuite returns default or manual Suite:
+func (p *PublishedRepo) GetSuite() string {
+	if p.Suite == "" {
+		return p.Distribution
+	}
+	return p.Suite
+}
+
 // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
 func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
 	collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool) error {
@@ -560,9 +588,17 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	}
 	defer os.RemoveAll(tempDir)
 
-	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash)
+	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash, p.SkipBz2)
 
 	legacyContentIndexes := map[string]*ContentsIndex{}
+	var count int64
+	for _, list := range lists {
+		count = count + int64(list.Len())
+	}
+
+	if progress != nil {
+		progress.InitBar(count, false, aptly.BarPublishGeneratePackageFiles)
+	}
 
 	for component, list := range lists {
 		hadUdebs := false
@@ -572,10 +608,6 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			indexes.PackageIndex(component, arch, false, false)
 		}
 
-		if progress != nil {
-			progress.InitBar(int64(list.Len()), false)
-		}
-
 		list.PrepareIndex()
 
 		contentIndexes := map[string]*ContentsIndex{}
@@ -612,8 +644,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			// to push each path of the package into the database.
 			// We'll want this batched so as to avoid an excessive
 			// amount of write() calls.
-			tempDB.StartBatch()
-			defer tempDB.FinishBatch()
+			batch := tempDB.CreateBatch()
 
 			for _, arch := range p.Architectures {
 				if pkg.MatchesArchitecture(arch) {
@@ -632,7 +663,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 								contentIndexesMap[key] = contentIndex
 							}
 
-							contentIndex.Push(qualifiedName, contents)
+							contentIndex.Push(qualifiedName, contents, batch)
 						}
 					}
 
@@ -657,7 +688,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			pkg.extra = nil
 			pkg.contents = nil
 
-			return nil
+			return batch.Write()
 		})
 
 		if err != nil {
@@ -684,10 +715,6 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			}
 		}
 
-		if progress != nil {
-			progress.ShutdownBar()
-		}
-
 		udebs := []bool{false}
 		if hadUdebs {
 			udebs = append(udebs, true)
@@ -707,6 +734,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				release["Component"] = component
 				release["Origin"] = p.GetOrigin()
 				release["Label"] = p.GetLabel()
+				release["Suite"] = p.GetSuite()
 				if p.AcquireByHash {
 					release["Acquire-By-Hash"] = "yes"
 				}
@@ -746,6 +774,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	}
 
 	if progress != nil {
+		progress.ShutdownBar()
 		progress.Printf("Finalizing metadata files...\n")
 	}
 
@@ -763,7 +792,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		release["ButAutomaticUpgrades"] = p.ButAutomaticUpgrades
 	}
 	release["Label"] = p.GetLabel()
-	release["Suite"] = p.Distribution
+	release["Suite"] = p.GetSuite()
 	release["Codename"] = p.Distribution
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
@@ -852,7 +881,6 @@ func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStor
 
 // PublishedRepoCollection does listing, updating/adding/deleting of PublishedRepos
 type PublishedRepoCollection struct {
-	*sync.RWMutex
 	db   database.Storage
 	list []*PublishedRepo
 }
@@ -860,8 +888,7 @@ type PublishedRepoCollection struct {
 // NewPublishedRepoCollection loads PublishedRepos from DB and makes up collection
 func NewPublishedRepoCollection(db database.Storage) *PublishedRepoCollection {
 	return &PublishedRepoCollection{
-		RWMutex: &sync.RWMutex{},
-		db:      db,
+		db: db,
 	}
 }
 
@@ -914,21 +941,16 @@ func (collection *PublishedRepoCollection) CheckDuplicate(repo *PublishedRepo) *
 }
 
 // Update stores updated information about repo in DB
-func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) (err error) {
-	err = collection.db.Put(repo.Key(), repo.Encode())
-	if err != nil {
-		return
-	}
+func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) error {
+	batch := collection.db.CreateBatch()
+	batch.Put(repo.Key(), repo.Encode())
 
 	if repo.SourceKind == SourceLocalRepo {
 		for component, item := range repo.sourceItems {
-			err = collection.db.Put(repo.RefKey(component), item.packageRefs.Encode())
-			if err != nil {
-				return
-			}
+			batch.Put(repo.RefKey(component), item.packageRefs.Encode())
 		}
 	}
-	return
+	return batch.Write()
 }
 
 // LoadComplete loads additional information for remote repo
@@ -1170,6 +1192,7 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress,
 	force, skipCleanup bool) error {
 
+	// TODO: load via transaction
 	collection.loadList()
 
 	repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
@@ -1221,17 +1244,12 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 		}
 	}
 
-	err = collection.db.Delete(repo.Key())
-	if err != nil {
-		return err
-	}
+	batch := collection.db.CreateBatch()
+	batch.Delete(repo.Key())
 
 	for _, component := range repo.Components() {
-		err = collection.db.Delete(repo.RefKey(component))
-		if err != nil {
-			return err
-		}
+		batch.Delete(repo.RefKey(component))
 	}
 
-	return nil
+	return batch.Write()
 }

deb/publish_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/ugorji/go/codec"
 
@@ -87,7 +88,7 @@ var _ = Suite(&PublishedRepoSuite{})
 func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.SetUpPackages()
 
-	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.factory = NewCollectionFactory(s.db)
 
 	s.root = c.MkDir()
@@ -449,7 +450,7 @@ type PublishedRepoCollectionSuite struct {
 var _ = Suite(&PublishedRepoCollectionSuite{})
 
 func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.factory = NewCollectionFactory(s.db)
 
 	s.snapshotCollection = s.factory.SnapshotCollection()
@@ -640,7 +641,7 @@ type PublishedRepoRemoveSuite struct {
 var _ = Suite(&PublishedRepoRemoveSuite{})
 
 func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
-	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.factory = NewCollectionFactory(s.db)
 
 	s.snapshotCollection = s.factory.SnapshotCollection()

deb/query.go

@@ -168,15 +168,15 @@ func (q *FieldQuery) Matches(pkg PackageLike) bool {
 	case VersionDontCare:
 		return field != ""
 	case VersionEqual:
-		return field == q.Value
+		return CompareVersions(field, q.Value) == 0
 	case VersionGreater:
-		return field > q.Value
+		return CompareVersions(field, q.Value) > 0
 	case VersionGreaterOrEqual:
-		return field >= q.Value
+		return CompareVersions(field, q.Value) >= 0
 	case VersionLess:
-		return field < q.Value
+		return CompareVersions(field, q.Value) < 0
 	case VersionLessOrEqual:
-		return field <= q.Value
+		return CompareVersions(field, q.Value) <= 0
 	case VersionPatternMatch:
 		matched, err := filepath.Match(q.Value, field)
 		return err == nil && matched

deb/query_test.go

@@ -0,0 +1,23 @@
+package deb
+
+import (
+	. "gopkg.in/check.v1"
+)
+
+type QuerySuite struct {
+}
+
+var _ = Suite(&QuerySuite{})
+
+func (s *QuerySuite) TestVersionCompare(c *C) {
+	q := FieldQuery{"Version", VersionLess, "5.0.0.2", nil}
+
+	p100 := Package{}
+	p100.Version = "5.0.0.100"
+
+	p1 := Package{}
+	p1.Version = "5.0.0.1"
+
+	c.Check(q.Matches(&p100), Equals, false)
+	c.Check(q.Matches(&p1), Equals, true)
+}

deb/reflist_test.go

@@ -3,7 +3,7 @@ package deb
 import (
 	"errors"
 
-	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
 
 	. "gopkg.in/check.v1"
 )
@@ -44,7 +44,7 @@ func (s *PackageRefListSuite) SetUpTest(c *C) {
 }
 
 func (s *PackageRefListSuite) TestNewPackageListFromRefList(c *C) {
-	db, _ := database.NewOpenDB(c.MkDir())
+	db, _ := goleveldb.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)
 	coll.Update(s.p1)
 	coll.Update(s.p3)
@@ -166,7 +166,7 @@ func (s *PackageRefListSuite) TestSubstract(c *C) {
 }
 
 func (s *PackageRefListSuite) TestDiff(c *C) {
-	db, _ := database.NewOpenDB(c.MkDir())
+	db, _ := goleveldb.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)
 
 	packages := []*Package{
@@ -238,7 +238,7 @@ func (s *PackageRefListSuite) TestDiff(c *C) {
 }
 
 func (s *PackageRefListSuite) TestMerge(c *C) {
-	db, _ := database.NewOpenDB(c.MkDir())
+	db, _ := goleveldb.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)
 
 	packages := []*Package{

deb/remote.go

@@ -12,7 +12,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 
@@ -33,7 +32,7 @@ const (
 
 // RemoteRepo represents remote (fetchable) Debian repository.
 //
-// Repostitory could be filtered when fetching by components, architectures
+// Repository could be filtered when fetching by components, architectures
 type RemoteRepo struct {
 	// Permanent internal ID
 	UUID string
@@ -52,7 +51,7 @@ type RemoteRepo struct {
 	// Last update date
 	LastDownloadDate time.Time
 	// Checksums for release files
-	ReleaseFiles map[string]utils.ChecksumInfo
+	ReleaseFiles map[string]utils.ChecksumInfo `json:"-"` // exclude from json output
 	// Filter for packages
 	Filter string
 	// Status marks state of repository (being updated, no action)
@@ -71,6 +70,8 @@ type RemoteRepo struct {
 	DownloadUdebs bool
 	// Should we download installer files?
 	DownloadInstaller bool
+	// Packages for json output
+	Packages []string `codec:"-" json:",omitempty"`
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 	// Parsed archived root
@@ -174,6 +175,11 @@ func (repo *RemoteRepo) RefList() *PackageRefList {
 	return repo.packageRefs
 }
 
+// PackageList returns package list for repo
+func (repo *RemoteRepo) PackageList() *PackageList {
+	return repo.packageList
+}
+
 // MarkAsUpdating puts current PID and sets status to updating
 func (repo *RemoteRepo) MarkAsUpdating() {
 	repo.Status = MirrorUpdating
@@ -423,7 +429,7 @@ ok:
 
 // DownloadPackageIndexes downloads & parses package index files
 func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, verifier pgp.Verifier, collectionFactory *CollectionFactory,
-	ignoreMismatch bool, maxTries int) error {
+	ignoreMismatch bool) error {
 	if repo.packageList != nil {
 		panic("packageList != nil")
 	}
@@ -456,7 +462,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 
 	for _, info := range packagesPaths {
 		path, kind, component, architecture := info[0], info[1], info[2], info[3]
-		packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries)
+		packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch)
 
 		isInstaller := kind == PackageTypeInstaller
 		if err != nil {
@@ -505,8 +511,10 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 		}
 		defer packagesFile.Close()
 
-		stat, _ := packagesFile.Stat()
-		progress.InitBar(stat.Size(), true)
+		if progress != nil {
+			stat, _ := packagesFile.Stat()
+			progress.InitBar(stat.Size(), true, aptly.BarMirrorUpdateBuildPackageList)
+		}
 
 		sreader := NewControlFileReader(packagesReader, false, isInstaller)
 
@@ -519,8 +527,10 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 				break
 			}
 
-			off, _ := packagesFile.Seek(0, 1)
-			progress.SetBar(int(off))
+			if progress != nil {
+				off, _ := packagesFile.Seek(0, 1)
+				progress.SetBar(int(off))
+			}
 
 			var p *Package
 
@@ -540,14 +550,20 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 				}
 			}
 			err = repo.packageList.Add(p)
-			if _, ok := err.(*PackageConflictError); ok {
-				progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
-			} else if err != nil {
-				return err
+			if err != nil {
+				if _, ok := err.(*PackageConflictError); ok {
+					if progress != nil {
+						progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
+					}
+				} else if err != nil {
+					return err
+				}
 			}
 		}
 
-		progress.ShutdownBar()
+		if progress != nil {
+			progress.ShutdownBar()
+		}
 	}
 
 	return nil
@@ -617,34 +633,44 @@ func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packag
 
 // FinalizeDownload swaps for final value of package refs
 func (repo *RemoteRepo) FinalizeDownload(collectionFactory *CollectionFactory, progress aptly.Progress) error {
+	transaction, err := collectionFactory.PackageCollection().db.OpenTransaction()
+	if err != nil {
+		return err
+	}
+	defer transaction.Discard()
+
 	repo.LastDownloadDate = time.Now()
 
 	if progress != nil {
-		progress.InitBar(int64(repo.packageList.Len()), true)
+		progress.InitBar(int64(repo.packageList.Len()), true, aptly.BarMirrorUpdateFinalizeDownload)
 	}
 
 	var i int
 
 	// update all the packages in collection
-	err := repo.packageList.ForEach(func(p *Package) error {
+	err = repo.packageList.ForEach(func(p *Package) error {
 		i++
 		if progress != nil {
 			progress.SetBar(i)
 		}
 		// download process might have updated checksums
 		p.UpdateFiles(p.Files())
-		return collectionFactory.PackageCollection().Update(p)
+		return collectionFactory.PackageCollection().UpdateInTransaction(p, transaction)
 	})
 
-	repo.packageRefs = NewPackageRefListFromPackageList(repo.packageList)
+	if err == nil {
+		repo.packageRefs = NewPackageRefListFromPackageList(repo.packageList)
+		repo.packageList = nil
+	}
 
 	if progress != nil {
 		progress.ShutdownBar()
 	}
 
-	repo.packageList = nil
-
-	return err
+	if err != nil {
+		return err
+	}
+	return transaction.Commit()
 }
 
 // Encode does msgpack encoding of RemoteRepo
@@ -696,6 +722,18 @@ func (repo *RemoteRepo) Decode(input []byte) error {
 			repo.ReleaseFiles = repo11.ReleaseFiles
 			repo.Filter = repo11.Filter
 			repo.FilterWithDeps = repo11.FilterWithDeps
+		} else if strings.Contains(err.Error(), "invalid length of bytes for decoding time") {
+			// DB created by old codec version, time.Time is not builtin type.
+			// https://github.com/ugorji/go-codec/issues/269
+			decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{
+				// only can be configured in Deprecated BasicHandle struct
+				BasicHandle: codec.BasicHandle{ // nolint: staticcheck
+					TimeNotBuiltin: true,
+				},
+			})
+			if err = decoder.Decode(repo); err != nil {
+				return err
+			}
 		} else {
 			return err
 		}
@@ -715,7 +753,6 @@ func (repo *RemoteRepo) RefKey() []byte {
 
 // RemoteRepoCollection does listing, updating/adding/deleting of RemoteRepos
 type RemoteRepoCollection struct {
-	*sync.RWMutex
 	db    database.Storage
 	cache map[string]*RemoteRepo
 }
@@ -723,9 +760,8 @@ type RemoteRepoCollection struct {
 // NewRemoteRepoCollection loads RemoteRepos from DB and makes up collection
 func NewRemoteRepoCollection(db database.Storage) *RemoteRepoCollection {
 	return &RemoteRepoCollection{
-		RWMutex: &sync.RWMutex{},
-		db:      db,
-		cache:   make(map[string]*RemoteRepo),
+		db:    db,
+		cache: make(map[string]*RemoteRepo),
 	}
 }
 
@@ -783,17 +819,13 @@ func (collection *RemoteRepoCollection) Add(repo *RemoteRepo) error {
 
 // Update stores updated information about repo in DB
 func (collection *RemoteRepoCollection) Update(repo *RemoteRepo) error {
-	err := collection.db.Put(repo.Key(), repo.Encode())
-	if err != nil {
-		return err
-	}
+	batch := collection.db.CreateBatch()
+
+	batch.Put(repo.Key(), repo.Encode())
 	if repo.packageRefs != nil {
-		err = collection.db.Put(repo.RefKey(), repo.packageRefs.Encode())
-		if err != nil {
-			return err
-		}
+		batch.Put(repo.RefKey(), repo.packageRefs.Encode())
 	}
-	return nil
+	return batch.Write()
 }
 
 // LoadComplete loads additional information for remote repo
@@ -866,16 +898,18 @@ func (collection *RemoteRepoCollection) Len() int {
 
 // Drop removes remote repo from collection
 func (collection *RemoteRepoCollection) Drop(repo *RemoteRepo) error {
-	if _, err := collection.db.Get(repo.Key()); err == database.ErrNotFound {
-		panic("repo not found!")
+	if _, err := collection.db.Get(repo.Key()); err != nil {
+		if err == database.ErrNotFound {
+			return errors.New("repo not found")
+		}
+
+		return err
 	}
 
 	delete(collection.cache, repo.UUID)
 
-	err := collection.db.Delete(repo.Key())
-	if err != nil {
-		return err
-	}
-
-	return collection.db.Delete(repo.RefKey())
+	batch := collection.db.CreateBatch()
+	batch.Delete(repo.Key())
+	batch.Delete(repo.RefKey())
+	return batch.Write()
 }

deb/remote_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/console"
 	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/http"
 	"github.com/aptly-dev/aptly/pgp"
@@ -92,7 +93,7 @@ func (s *RemoteRepoSuite) SetUpTest(c *C) {
 	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false)
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress()
-	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.collectionFactory = NewCollectionFactory(s.db)
 	s.packagePool = files.NewPackagePool(c.MkDir(), false)
 	s.cs = files.NewMockChecksumStorage()
@@ -276,7 +277,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -303,7 +304,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -324,7 +325,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -351,7 +352,7 @@ func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) {
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/installer-i386/current/images/SHA256SUMS", exampleInstallerHashSumFile)
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST", exampleInstallerManifestFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -395,7 +396,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -439,7 +440,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -464,7 +465,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -488,7 +489,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -516,7 +517,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -538,7 +539,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -569,7 +570,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -615,7 +616,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -641,7 +642,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -663,7 +664,7 @@ type RemoteRepoCollectionSuite struct {
 var _ = Suite(&RemoteRepoCollectionSuite{})
 
 func (s *RemoteRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.collection = NewRemoteRepoCollection(s.db)
 	s.SetUpPackages()
 }
@@ -773,7 +774,7 @@ func (s *RemoteRepoCollectionSuite) TestDrop(c *C) {
 	r2, _ := collection.ByName("tyndex")
 	c.Check(r2.String(), Equals, repo2.String())
 
-	c.Check(func() { s.collection.Drop(repo1) }, Panics, "repo not found!")
+	c.Check(s.collection.Drop(repo1), ErrorMatches, "repo not found")
 }
 
 const exampleReleaseFile = `Origin: LP-PPA-agenda-developers-daily

deb/snapshot.go

@@ -7,7 +7,6 @@ import (
 	"log"
 	"sort"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/aptly-dev/aptly/database"
@@ -19,15 +18,21 @@ import (
 // Snapshot is immutable state of repository: list of packages
 type Snapshot struct {
 	// Persisten internal ID
-	UUID string `json:"-"`
+	UUID string `codec:"UUID" json:"-"`
 	// Human-readable name
 	Name string
 	// Date of creation
 	CreatedAt time.Time
 
 	// Source: kind + ID
-	SourceKind string   `json:"-"`
-	SourceIDs  []string `json:"-"`
+	SourceKind string   `codec:"SourceKind"`
+	SourceIDs  []string `codec:"SourceIDs" json:"-"`
+	// Sources
+	Snapshots   []*Snapshot   `codec:"-" json:",omitempty"`
+	RemoteRepos []*RemoteRepo `codec:"-" json:",omitempty"`
+	LocalRepos  []*LocalRepo  `codec:"-" json:",omitempty"`
+	Packages    []string      `codec:"-" json:",omitempty"`
+
 	// Description of how snapshot was created
 	Description string
 
@@ -120,6 +125,13 @@ func (s *Snapshot) Key() []byte {
 	return []byte("S" + s.UUID)
 }
 
+// ResourceKey is a unique identifier of the resource
+// this snapshot uses. Instead of uuid it uses name
+// which needs to be unique as well.
+func (s *Snapshot) ResourceKey() []byte {
+	return []byte("S" + s.Name)
+}
+
 // RefKey is a unique id for package reference list
 func (s *Snapshot) RefKey() []byte {
 	return []byte("E" + s.UUID)
@@ -163,6 +175,18 @@ func (s *Snapshot) Decode(input []byte) error {
 			s.SourceKind = snapshot11.SourceKind
 			s.SourceIDs = snapshot11.SourceIDs
 			s.Description = snapshot11.Description
+		} else if strings.Contains(err.Error(), "invalid length of bytes for decoding time") {
+			// DB created by old codec version, time.Time is not builtin type.
+			// https://github.com/ugorji/go-codec/issues/269
+			decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{
+				// only can be configured in Deprecated BasicHandle struct
+				BasicHandle: codec.BasicHandle{ // nolint: staticcheck
+					TimeNotBuiltin: true,
+				},
+			})
+			if err = decoder.Decode(s); err != nil {
+				return err
+			}
 		} else {
 			return err
 		}
@@ -172,7 +196,6 @@ func (s *Snapshot) Decode(input []byte) error {
 
 // SnapshotCollection does listing, updating/adding/deleting of Snapshots
 type SnapshotCollection struct {
-	*sync.RWMutex
 	db    database.Storage
 	cache map[string]*Snapshot
 }
@@ -180,9 +203,8 @@ type SnapshotCollection struct {
 // NewSnapshotCollection loads Snapshots from DB and makes up collection
 func NewSnapshotCollection(db database.Storage) *SnapshotCollection {
 	return &SnapshotCollection{
-		RWMutex: &sync.RWMutex{},
-		db:      db,
-		cache:   map[string]*Snapshot{},
+		db:    db,
+		cache: map[string]*Snapshot{},
 	}
 }
 
@@ -204,14 +226,14 @@ func (collection *SnapshotCollection) Add(snapshot *Snapshot) error {
 
 // Update stores updated information about snapshot in DB
 func (collection *SnapshotCollection) Update(snapshot *Snapshot) error {
-	err := collection.db.Put(snapshot.Key(), snapshot.Encode())
-	if err != nil {
-		return err
-	}
+	batch := collection.db.CreateBatch()
+
+	batch.Put(snapshot.Key(), snapshot.Encode())
 	if snapshot.packageRefs != nil {
-		return collection.db.Put(snapshot.RefKey(), snapshot.packageRefs.Encode())
+		batch.Put(snapshot.RefKey(), snapshot.packageRefs.Encode())
 	}
-	return nil
+
+	return batch.Write()
 }
 
 // LoadComplete loads additional information about snapshot
@@ -367,18 +389,20 @@ func (collection *SnapshotCollection) Len() int {
 
 // Drop removes snapshot from collection
 func (collection *SnapshotCollection) Drop(snapshot *Snapshot) error {
-	if _, err := collection.db.Get(snapshot.Key()); err == database.ErrNotFound {
-		panic("snapshot not found!")
+	if _, err := collection.db.Get(snapshot.Key()); err != nil {
+		if err == database.ErrNotFound {
+			return errors.New("snapshot not found")
+		}
+
+		return err
 	}
 
 	delete(collection.cache, snapshot.UUID)
 
-	err := collection.db.Delete(snapshot.Key())
-	if err != nil {
-		return err
-	}
-
-	return collection.db.Delete(snapshot.RefKey())
+	batch := collection.db.CreateBatch()
+	batch.Delete(snapshot.Key())
+	batch.Delete(snapshot.RefKey())
+	return batch.Write()
 }
 
 // Snapshot sorting methods

deb/snapshot_bench_test.go

@@ -5,7 +5,7 @@ import (
 	"os"
 	"testing"
 
-	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
 )
 
 func BenchmarkSnapshotCollectionForEach(b *testing.B) {
@@ -14,7 +14,7 @@ func BenchmarkSnapshotCollectionForEach(b *testing.B) {
 	tmpDir := os.TempDir()
 	defer os.RemoveAll(tmpDir)
 
-	db, _ := database.NewOpenDB(tmpDir)
+	db, _ := goleveldb.NewOpenDB(tmpDir)
 	defer db.Close()
 
 	collection := NewSnapshotCollection(db)
@@ -43,7 +43,7 @@ func BenchmarkSnapshotCollectionByUUID(b *testing.B) {
 	tmpDir := os.TempDir()
 	defer os.RemoveAll(tmpDir)
 
-	db, _ := database.NewOpenDB(tmpDir)
+	db, _ := goleveldb.NewOpenDB(tmpDir)
 	defer db.Close()
 
 	collection := NewSnapshotCollection(db)
@@ -74,7 +74,7 @@ func BenchmarkSnapshotCollectionByName(b *testing.B) {
 	tmpDir := os.TempDir()
 	defer os.RemoveAll(tmpDir)
 
-	db, _ := database.NewOpenDB(tmpDir)
+	db, _ := goleveldb.NewOpenDB(tmpDir)
 	defer db.Close()
 
 	collection := NewSnapshotCollection(db)

deb/snapshot_test.go

@@ -5,6 +5,7 @@ import (
 	"sort"
 
 	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
 
 	. "gopkg.in/check.v1"
 )
@@ -113,7 +114,7 @@ type SnapshotCollectionSuite struct {
 var _ = Suite(&SnapshotCollectionSuite{})
 
 func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.collection = NewSnapshotCollection(s.db)
 	s.SetUpPackages()
 
@@ -279,5 +280,5 @@ func (s *SnapshotCollectionSuite) TestDrop(c *C) {
 	_, err = collection.ByUUID(s.snapshot1.UUID)
 	c.Check(err, ErrorMatches, "snapshot .* not found")
 
-	c.Check(func() { s.collection.Drop(s.snapshot1) }, Panics, "snapshot not found!")
+	c.Check(s.collection.Drop(s.snapshot1), ErrorMatches, "snapshot not found")
 }

deb/testdata/dbgsym-with-source-version/dbgsym-with-source-version_2021.01.dsc

@@ -0,0 +1,16 @@
+Format: 3.0 (native)
+Source: dbgsym-with-source-version
+Binary: dbgsym-with-source-version
+Architecture: any
+Version: 2021.01
+Maintainer: Ratchanan Srirattanamet <ratchanan@ubports.com>
+Standards-Version: 4.4.1
+Build-Depends: debhelper-compat (= 12)
+Package-List:
+ dbgsym-with-source-version deb unknown optional arch=any
+Checksums-Sha1:
+ 626c500200d9837b475f61a2563a647e785a9ccf 1664 dbgsym-with-source-version_2021.01.tar.xz
+Checksums-Sha256:
+ 00e1915300d0996056867b66d2cfc608e3839febb9feb596fe940f99ced632b0 1664 dbgsym-with-source-version_2021.01.tar.xz
+Files:
+ 5f50d8d87421226d33536471d294140e 1664 dbgsym-with-source-version_2021.01.tar.xz

deb/testdata/dbgsym-with-source-version/dbgsym-with-source-version_2021.01_amd64.buildinfo

@@ -0,0 +1,182 @@
+Format: 1.0
+Source: dbgsym-with-source-version
+Binary: dbgsym-with-source-version
+Architecture: amd64 source
+Version: 2021.01
+Checksums-Md5:
+ d411f22d5ae7a68256faa2d02276da33 646 dbgsym-with-source-version_2021.01.dsc
+ 91adadcd5b04c89a6775e6be20d03705 2668 dbgsym-with-source-version-dbgsym_0.1+aptly2021.01_amd64.ddeb
+ 1e3c04499395851894fda69903b1c6ff 3520 dbgsym-with-source-version_0.1+aptly2021.01_amd64.deb
+Checksums-Sha1:
+ 07b21e2cf16c966695455a56343f653b0322d00c 646 dbgsym-with-source-version_2021.01.dsc
+ c8b0e5ef412749c60ada84f07b9a9abeb8a9b58f 2668 dbgsym-with-source-version-dbgsym_0.1+aptly2021.01_amd64.ddeb
+ e46bb74e4ca1a8b6013b145d95db90264b873225 3520 dbgsym-with-source-version_0.1+aptly2021.01_amd64.deb
+Checksums-Sha256:
+ f1d6a9ab1b73d8ed798ebac79b472433712cc964fa73c60fc4b82e828a279070 646 dbgsym-with-source-version_2021.01.dsc
+ afba2bf24c4978de82d49dc196ee26ed678ef54386fa4f1982f70aec985ccc02 2668 dbgsym-with-source-version-dbgsym_0.1+aptly2021.01_amd64.ddeb
+ 7834699e4f3f27ed7cb8776e6846e8367d6e18d0c589bb283900841a2a5bbcd5 3520 dbgsym-with-source-version_0.1+aptly2021.01_amd64.deb
+Build-Origin: Ubuntu
+Build-Architecture: amd64
+Build-Date: Fri, 08 Jan 2021 20:59:06 +0700
+Build-Tainted-By:
+ usr-local-has-libraries
+ usr-local-has-programs
+Installed-Build-Depends:
+ autoconf (= 2.69-11.1),
+ automake (= 1:1.16.1-4ubuntu6),
+ autopoint (= 0.19.8.1-10build1),
+ autotools-dev (= 20180224.1),
+ base-files (= 11ubuntu5.2),
+ base-passwd (= 3.5.47),
+ bash (= 5.0-6ubuntu1.1),
+ binutils (= 2.34-6ubuntu1),
+ binutils-common (= 2.34-6ubuntu1),
+ binutils-x86-64-linux-gnu (= 2.34-6ubuntu1),
+ bsdmainutils (= 11.1.2ubuntu3),
+ bsdutils (= 1:2.34-0.1ubuntu9.1),
+ build-essential (= 12.8ubuntu1.1),
+ bzip2 (= 1.0.8-2),
+ coreutils (= 8.30-3ubuntu2),
+ cpp (= 4:9.3.0-1ubuntu2),
+ cpp-7 (= 7.5.0-6ubuntu2),
+ cpp-9 (= 9.3.0-17ubuntu1~20.04),
+ dash (= 0.5.10.2-6),
+ debconf (= 1.5.73),
+ debhelper (= 12.10ubuntu1),
+ debianutils (= 4.9.1),
+ dh-autoreconf (= 19),
+ dh-strip-nondeterminism (= 1.7.0-1),
+ diffutils (= 1:3.7-3),
+ dpkg (= 1.19.7ubuntu3),
+ dpkg-dev (= 1.19.7ubuntu3),
+ dwz (= 0.13-5),
+ file (= 1:5.38-4),
+ findutils (= 4.7.0-1ubuntu1),
+ g++ (= 4:9.3.0-1ubuntu2),
+ g++-9 (= 9.3.0-17ubuntu1~20.04),
+ gawk (= 1:5.0.1+dfsg-1),
+ gcc (= 4:9.3.0-1ubuntu2),
+ gcc-10-base (= 10.2.0-5ubuntu1~20.04),
+ gcc-7 (= 7.5.0-6ubuntu2),
+ gcc-7-base (= 7.5.0-6ubuntu2),
+ gcc-8-base (= 8.4.0-3ubuntu2),
+ gcc-9 (= 9.3.0-17ubuntu1~20.04),
+ gcc-9-base (= 9.3.0-17ubuntu1~20.04),
+ gettext (= 0.19.8.1-10build1),
+ gettext-base (= 0.19.8.1-10build1),
+ grep (= 3.4-1),
+ groff-base (= 1.22.4-4build1),
+ gzip (= 1.10-0ubuntu4),
+ hostname (= 3.23),
+ init-system-helpers (= 1.57),
+ install-info (= 6.7.0.dfsg.2-5),
+ intltool-debian (= 0.35.0+20060710.5),
+ libacl1 (= 2.2.53-6),
+ libarchive-zip-perl (= 1.67-2),
+ libasan4 (= 7.5.0-6ubuntu2),
+ libasan5 (= 9.3.0-17ubuntu1~20.04),
+ libatomic1 (= 10.2.0-5ubuntu1~20.04),
+ libattr1 (= 1:2.4.48-5),
+ libaudit-common (= 1:2.8.5-2ubuntu6),
+ libaudit1 (= 1:2.8.5-2ubuntu6),
+ libbinutils (= 2.34-6ubuntu1),
+ libblkid1 (= 2.34-0.1ubuntu9.1),
+ libbsd0 (= 0.10.0-1),
+ libbz2-1.0 (= 1.0.8-2),
+ libc-bin (= 2.31-0ubuntu9.1),
+ libc-dev-bin (= 2.31-0ubuntu9.1),
+ libc6 (= 2.31-0ubuntu9.1),
+ libc6-dev (= 2.31-0ubuntu9.1),
+ libcap-ng0 (= 0.7.9-2.1build1),
+ libcc1-0 (= 10.2.0-5ubuntu1~20.04),
+ libcilkrts5 (= 7.5.0-6ubuntu2),
+ libcroco3 (= 0.6.13-1),
+ libcrypt-dev (= 1:4.4.10-10ubuntu4),
+ libcrypt1 (= 1:4.4.10-10ubuntu4),
+ libctf-nobfd0 (= 2.34-6ubuntu1),
+ libctf0 (= 2.34-6ubuntu1),
+ libdb5.3 (= 5.3.28+dfsg1-0.6ubuntu2),
+ libdebconfclient0 (= 0.251ubuntu1),
+ libdebhelper-perl (= 12.10ubuntu1),
+ libdpkg-perl (= 1.19.7ubuntu3),
+ libelf1 (= 0.176-1.1build1),
+ libffi7 (= 3.3-4),
+ libfile-stripnondeterminism-perl (= 1.7.0-1),
+ libgcc-7-dev (= 7.5.0-6ubuntu2),
+ libgcc-9-dev (= 9.3.0-17ubuntu1~20.04),
+ libgcc-s1 (= 10.2.0-5ubuntu1~20.04),
+ libgcrypt20 (= 1.8.5-5ubuntu1),
+ libgdbm-compat4 (= 1.18.1-5),
+ libgdbm6 (= 1.18.1-5),
+ libglib2.0-0 (= 2.64.3-1~ubuntu20.04.1),
+ libgmp10 (= 2:6.2.0+dfsg-4),
+ libgomp1 (= 10.2.0-5ubuntu1~20.04),
+ libgpg-error0 (= 1.37-1),
+ libicu66 (= 66.1-2ubuntu2),
+ libisl22 (= 0.22.1-1),
+ libitm1 (= 10.2.0-5ubuntu1~20.04),
+ liblsan0 (= 10.2.0-5ubuntu1~20.04),
+ liblz4-1 (= 1.9.2-2),
+ liblzma5 (= 5.2.4-1ubuntu1),
+ libmagic-mgc (= 1:5.38-4),
+ libmagic1 (= 1:5.38-4),
+ libmount1 (= 2.34-0.1ubuntu9.1),
+ libmpc3 (= 1.1.0-1),
+ libmpfr6 (= 4.0.2-1),
+ libmpx2 (= 8.4.0-3ubuntu2),
+ libpam-modules (= 1.3.1-5ubuntu4.1),
+ libpam-modules-bin (= 1.3.1-5ubuntu4.1),
+ libpam-runtime (= 1.3.1-5ubuntu4.1),
+ libpam0g (= 1.3.1-5ubuntu4.1),
+ libpcre2-8-0 (= 10.34-7),
+ libpcre3 (= 2:8.39-12build1),
+ libperl5.30 (= 5.30.0-9ubuntu0.2),
+ libpipeline1 (= 1.5.2-2build1),
+ libquadmath0 (= 10.2.0-5ubuntu1~20.04),
+ libreadline8 (= 8.0-4),
+ libseccomp2 (= 2.4.3-1ubuntu3.20.04.3),
+ libselinux1 (= 3.0-1build2),
+ libsigsegv2 (= 2.12-2),
+ libsmartcols1 (= 2.34-0.1ubuntu9.1),
+ libstdc++-9-dev (= 9.3.0-17ubuntu1~20.04),
+ libstdc++6 (= 10.2.0-5ubuntu1~20.04),
+ libsub-override-perl (= 0.09-2),
+ libsystemd0 (= 245.4-4ubuntu3.3),
+ libtinfo6 (= 6.2-0ubuntu2),
+ libtool (= 2.4.6-14),
+ libtsan0 (= 10.2.0-5ubuntu1~20.04),
+ libubsan0 (= 7.5.0-6ubuntu2),
+ libubsan1 (= 10.2.0-5ubuntu1~20.04),
+ libuchardet0 (= 0.0.6-3build1),
+ libudev1 (= 245.4-4ubuntu3.3),
+ libunistring2 (= 0.9.10-2),
+ libuuid1 (= 2.34-0.1ubuntu9.1),
+ libxml2 (= 2.9.10+dfsg-5),
+ libzstd1 (= 1.4.4+dfsg-3),
+ linux-libc-dev (= 5.4.0-60.67),
+ login (= 1:4.8.1-1ubuntu5.20.04),
+ lsb-base (= 11.1.0ubuntu2),
+ m4 (= 1.4.18-4),
+ make (= 4.2.1-1.2),
+ man-db (= 2.9.1-1),
+ mawk (= 1.3.4.20200120-2),
+ ncurses-base (= 6.2-0ubuntu2),
+ ncurses-bin (= 6.2-0ubuntu2),
+ patch (= 2.7.6-6),
+ perl (= 5.30.0-9ubuntu0.2),
+ perl-base (= 5.30.0-9ubuntu0.2),
+ perl-modules-5.30 (= 5.30.0-9ubuntu0.2),
+ po-debconf (= 1.0.21),
+ readline-common (= 8.0-4),
+ sed (= 4.7-1),
+ sensible-utils (= 0.0.12+nmu1),
+ sysvinit-utils (= 2.96-2.1ubuntu1),
+ tar (= 1.30+dfsg-7),
+ tzdata (= 2020f-0ubuntu0.20.04),
+ util-linux (= 2.34-0.1ubuntu9.1),
+ xz-utils (= 5.2.4-1ubuntu1),
+ zlib1g (= 1:1.2.11.dfsg-2ubuntu1.2)
+Environment:
+ DEB_BUILD_OPTIONS="parallel=4"
+ LANG="th_TH.UTF-8"
+ SOURCE_DATE_EPOCH="1610111766"

deb/testdata/dbgsym-with-source-version/dbgsym-with-source-version_2021.01_amd64.changes

@@ -0,0 +1,34 @@
+Format: 1.8
+Date: Fri, 08 Jan 2021 20:16:06 +0700
+Source: dbgsym-with-source-version
+Binary: dbgsym-with-source-version
+Architecture: source amd64
+Version: 2021.01
+Distribution: unstable
+Urgency: medium
+Maintainer: Ratchanan Srirattanamet <ratchanan@ubports.com>
+Changed-By: Ratchanan Srirattanamet <ratchanan@ubports.com>
+Description:
+ dbgsym-with-source-version - Test Aptly importing dbgsym with versioned Source field
+Changes:
+ dbgsym-with-source-version (2021.01) unstable; urgency=medium
+ .
+   * Initial Release.
+Checksums-Sha1:
+ 07b21e2cf16c966695455a56343f653b0322d00c 646 dbgsym-with-source-version_2021.01.dsc
+ 626c500200d9837b475f61a2563a647e785a9ccf 1664 dbgsym-with-source-version_2021.01.tar.xz
+ c8b0e5ef412749c60ada84f07b9a9abeb8a9b58f 2668 dbgsym-with-source-version-dbgsym_0.1+aptly2021.01_amd64.ddeb
+ e46bb74e4ca1a8b6013b145d95db90264b873225 3520 dbgsym-with-source-version_0.1+aptly2021.01_amd64.deb
+ d95a8d9cf1e7c0e68a99ad901ed88e8ac2f22fc0 6155 dbgsym-with-source-version_2021.01_amd64.buildinfo
+Checksums-Sha256:
+ f1d6a9ab1b73d8ed798ebac79b472433712cc964fa73c60fc4b82e828a279070 646 dbgsym-with-source-version_2021.01.dsc
+ 00e1915300d0996056867b66d2cfc608e3839febb9feb596fe940f99ced632b0 1664 dbgsym-with-source-version_2021.01.tar.xz
+ afba2bf24c4978de82d49dc196ee26ed678ef54386fa4f1982f70aec985ccc02 2668 dbgsym-with-source-version-dbgsym_0.1+aptly2021.01_amd64.ddeb
+ 7834699e4f3f27ed7cb8776e6846e8367d6e18d0c589bb283900841a2a5bbcd5 3520 dbgsym-with-source-version_0.1+aptly2021.01_amd64.deb
+ 763abc57edbe79e2a7cfc6665bcf63b2c58d8a3d04fb27d7247f0272e21032e1 6155 dbgsym-with-source-version_2021.01_amd64.buildinfo
+Files:
+ d411f22d5ae7a68256faa2d02276da33 646 unknown optional dbgsym-with-source-version_2021.01.dsc
+ 5f50d8d87421226d33536471d294140e 1664 unknown optional dbgsym-with-source-version_2021.01.tar.xz
+ 91adadcd5b04c89a6775e6be20d03705 2668 debug optional dbgsym-with-source-version-dbgsym_0.1+aptly2021.01_amd64.ddeb
+ 1e3c04499395851894fda69903b1c6ff 3520 unknown optional dbgsym-with-source-version_0.1+aptly2021.01_amd64.deb
+ 62d96c0271d499d9c848ec66fc888f6d 6155 unknown optional dbgsym-with-source-version_2021.01_amd64.buildinfo

deb/version.go

@@ -236,6 +236,17 @@ func ParseDependencyVariants(variants string) (l []Dependency, err error) {
 	return
 }
 
+// ParseDependencyArch parses the dependency name in format "pkg:any" into name and architecture
+func ParseDependencyArch(d *Dependency) {
+	if strings.ContainsRune(d.Pkg, ':') {
+		parts := strings.SplitN(d.Pkg, ":", 2)
+		d.Pkg, d.Architecture = parts[0], parts[1]
+		if d.Architecture == "any" {
+			d.Architecture = ""
+		}
+	}
+}
+
 // ParseDependency parses dependency in format "pkg (>= 1.35) [arch]" into parts
 func ParseDependency(dep string) (d Dependency, err error) {
 	if strings.HasSuffix(dep, "}") {
@@ -252,6 +263,7 @@ func ParseDependency(dep string) (d Dependency, err error) {
 	if !strings.HasSuffix(dep, ")") {
 		d.Pkg = strings.TrimSpace(dep)
 		d.Relation = VersionDontCare
+		ParseDependencyArch(&d)
 		return
 	}
 
@@ -262,13 +274,7 @@ func ParseDependency(dep string) (d Dependency, err error) {
 	}
 
 	d.Pkg = strings.TrimSpace(dep[0:i])
-	if strings.ContainsRune(d.Pkg, ':') {
-		parts := strings.SplitN(d.Pkg, ":", 2)
-		d.Pkg, d.Architecture = parts[0], parts[1]
-		if d.Architecture == "any" {
-			d.Architecture = ""
-		}
-	}
+	ParseDependencyArch(&d)
 
 	rel := ""
 	if dep[i+1] == '>' || dep[i+1] == '<' || dep[i+1] == '=' {

deb/version_test.go

@@ -98,6 +98,8 @@ func (s *VersionSuite) TestCompareVersions(c *C) {
 
 	c.Check(CompareVersions("1.0-133-avc", "1.1"), Equals, -1)
 	c.Check(CompareVersions("1.0-133-avc", "1.0"), Equals, 1)
+
+	c.Check(CompareVersions("5.2.0.3", "5.2.0.283"), Equals, -1)
 }
 
 func (s *VersionSuite) TestParseDependency(c *C) {
@@ -232,4 +234,8 @@ func (s *VersionSuite) TestDependencyString(c *C) {
 	d, _ = ParseDependency("dpkg")
 	d.Architecture = "i386"
 	c.Check(d.String(), Equals, "dpkg [i386]")
+
+	d, _ = ParseDependency("dpkg:any")
+	c.Check(d.Pkg, Equals, "dpkg")
+	c.Check(d.Architecture, Equals, "")
 }