Version bump to 0.9.5.

published repositories even if cleanup fails. #153

Makefile

@@ -67,7 +67,7 @@ package:
 	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
 	gzip root/usr/share/man/man1/aptly.1
 	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
-	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 -C root/ .
+	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 --deb-recommends graphviz -C root/ .
 	mv aptly_$(VERSION)_*.deb ~
 
 src-package:

api/publish.go

@@ -312,6 +312,8 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 
 // DELETE /publish/:prefix/:distribution
 func apiPublishDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+
 	param := parseEscapedPath(c.Params.ByName("prefix"))
 	storage, prefix := deb.ParsePrefix(param)
 	distribution := c.Params.ByName("distribution")
@@ -326,7 +328,7 @@ func apiPublishDrop(c *gin.Context) {
 	defer collection.Unlock()
 
 	err := collection.Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress())
+		context.CollectionFactory(), context.Progress(), force)
 	if err != nil {
 		c.Fail(500, fmt.Errorf("unable to drop: %s", err))
 		return

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
 // Version of aptly
-const Version = "0.9"
+const Version = "0.9.5"
 
 // Enable debugging features?
 const EnableDebug = false

cmd/db_cleanup.go

@@ -6,6 +6,7 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"sort"
+	"strings"
 )
 
 // aptly db cleanup
@@ -17,51 +18,112 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
+	verbose := context.Flags().Lookup("verbose").Value.Get().(bool)
+	dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
+
 	// collect information about references packages...
 	existingPackageRefs := deb.NewPackageRefList()
 
-	context.Progress().Printf("Loading mirrors, local repos, snapshots and published repos...\n")
+	// used only in verbose mode to report package use source
+	packageRefSources := map[string][]string{}
+
+	context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|")
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
+	}
 	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
 		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
 		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("mirror %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
+
 		return nil
 	})
 	if err != nil {
 		return err
 	}
 
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
+	}
 	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
 		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
+
 		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("local repo %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
+
 		return nil
 	})
 	if err != nil {
 		return err
 	}
 
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
+	}
 	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
+		}
+
 		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return err
 		}
-		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false)
+
+		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+
+		if verbose {
+			description := fmt.Sprintf("snapshot %s", snapshot.Name)
+			snapshot.RefList().ForEach(func(key []byte) error {
+				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+				return nil
+			})
+		}
 		return nil
 	})
 	if err != nil {
 		return err
 	}
 
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
+	}
 	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
+		}
 		if published.SourceKind != "local" {
 			return nil
 		}
@@ -71,7 +133,15 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		}
 
 		for _, component := range published.Components() {
-			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false)
+			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+			if verbose {
+				description := fmt.Sprintf("published repository %s:%s/%s component %s",
+					published.Storage, published.Prefix, published.Distribution, component)
+				published.RefList(component).ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
 		return nil
 	})
@@ -80,38 +150,65 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	}
 
 	// ... and compare it to the list of all packages
-	context.Progress().Printf("Loading list of all packages...\n")
+	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
 	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
 
 	toDelete := allPackageRefs.Substract(existingPackageRefs)
 
 	// delete packages that are no longer referenced
-	context.Progress().Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())
 
 	// database can't err as collection factory already constructed
 	db, _ := context.Database()
-	db.StartBatch()
-	err = toDelete.ForEach(func(ref []byte) error {
-		return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
-	})
-	if err != nil {
-		return err
-	}
 
-	err = db.FinishBatch()
+	if toDelete.Len() > 0 {
-	if err != nil {
+		if verbose {
-		return fmt.Errorf("unable to write to DB: %s", err)
+			context.Progress().ColoredPrintf("@{r}List of package keys to delete:@|")
+			err = toDelete.ForEach(func(ref []byte) error {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", string(ref))
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if !dryRun {
+			db.StartBatch()
+			err = toDelete.ForEach(func(ref []byte) error {
+				return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
+			})
+			if err != nil {
+				return err
+			}
+
+			err = db.FinishBatch()
+			if err != nil {
+				return fmt.Errorf("unable to write to DB: %s", err)
+			}
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped deletion, as -dry-run has been requested.@|")
+		}
 	}
 
 	// now, build a list of files that should be present in Repository (package pool)
-	context.Progress().Printf("Building list of files referenced by packages...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())
 	context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
 
 	err = existingPackageRefs.ForEach(func(key []byte) error {
 		pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
 		if err2 != nil {
-			return err2
+			tail := ""
+			if verbose {
+				tail = fmt.Sprintf(" (sources: %s)", strings.Join(packageRefSources[string(key)], ", "))
+			}
+			if dryRun {
+				context.Progress().ColoredPrintf("@{r!}Unresolvable package reference, skipping (-dry-run): %s: %s%s",
+					string(key), err2, tail)
+				return nil
+			}
+			return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
 		}
 		paths, err2 := pkg.FilepathList(context.PackagePool())
 		if err2 != nil {
@@ -130,7 +227,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	context.Progress().ShutdownBar()
 
 	// build a list of files in the package pool
-	context.Progress().Printf("Building list of files in package pool...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files in package pool...@|")
 	existingFiles, err := context.PackagePool().FilepathList(context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to collect file paths: %s", err)
@@ -140,28 +237,43 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
 
 	// delete files that are no longer referenced
-	context.Progress().Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced files (%d)...@|", len(filesToDelete))
 
 	if len(filesToDelete) > 0 {
-		context.Progress().InitBar(int64(len(filesToDelete)), false)
+		if verbose {
-
+			context.Progress().ColoredPrintf("@{r}List of files to be deleted:@|")
-		var size, totalSize int64
+			for _, file := range filesToDelete {
-		for _, file := range filesToDelete {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", file)
-			size, err = context.PackagePool().Remove(file)
-			if err != nil {
-				return err
 			}
-
-			context.Progress().AddBar(1)
-			totalSize += size
 		}
-		context.Progress().ShutdownBar()
 
-		context.Progress().Printf("Disk space freed: %s...\n", utils.HumanBytes(totalSize))
+		if !dryRun {
+			context.Progress().InitBar(int64(len(filesToDelete)), false)
+
+			var size, totalSize int64
+			for _, file := range filesToDelete {
+				size, err = context.PackagePool().Remove(file)
+				if err != nil {
+					return err
+				}
+
+				context.Progress().AddBar(1)
+				totalSize += size
+			}
+			context.Progress().ShutdownBar()
+
+			context.Progress().ColoredPrintf("@{w!}Disk space freed: %s...@|", utils.HumanBytes(totalSize))
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped file deletion, as -dry-run has been requested.@|")
+		}
 	}
 
-	context.Progress().Printf("Compacting database...\n")
+	if !dryRun {
-	err = db.CompactDB()
+		context.Progress().ColoredPrintf("@{w!}Compacting database...@|")
+		err = db.CompactDB()
+	} else {
+		context.Progress().ColoredPrintf("@{y!}Skipped DB compaction, as -dry-run has been requested.@|")
+	}
 
 	return err
 }
@@ -181,5 +293,8 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("verbose", false, "be verbose when loading objects/removing them")
+	cmd.Flag.Bool("dry-run", false, "don't delete anything")
+
 	return cmd
 }

cmd/publish_drop.go

@@ -23,7 +23,7 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 	storage, prefix := deb.ParsePrefix(param)
 
 	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress())
+		context.CollectionFactory(), context.Progress(), context.Flags().Lookup("force-drop").Value.Get().(bool))
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -48,5 +48,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
+
 	return cmd
 }

cmd/snapshot_merge.go

@@ -39,7 +39,7 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 
 	result := sources[0].RefList()
 	for i := 1; i < len(sources); i++ {
-		result = result.Merge(sources[i].RefList(), overrideMatching)
+		result = result.Merge(sources[i].RefList(), overrideMatching, false)
 	}
 
 	if latest {

cmd/snapshot_search.go

@@ -96,6 +96,10 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		return fmt.Errorf("unable to search: %s", err)
 	}
 
+	if result.Len() == 0 {
+		return fmt.Errorf("no results")
+	}
+
 	result.ForEach(func(p *deb.Package) error {
 		context.Progress().Printf("%s\n", p)
 		return nil

console/progress.go

@@ -133,19 +133,30 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
 		p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"}
 	} else {
 		// stip color marks
-		var prev rune
+		var inColorMark, inCurly bool
 		msg = strings.Map(func(r rune) rune {
-			if prev == '@' {
+			if inColorMark {
-				prev = 0
+				if inCurly {
-				if r == '@' {
+					if r == '}' {
-					return r
+						inCurly = false
+						inColorMark = false
+						return -1
+					}
+				} else {
+					if r == '{' {
+						inCurly = true
+					} else if r == '@' {
+						return '@'
+					} else {
+						inColorMark = false
+					}
 				}
 				return -1
 			}
-			prev = r
-			if r == '@' {
-				return -1
 
+			if r == '@' {
+				inColorMark = true
+				return -1
 			}
 
 			return r

deb/format.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"io"
 	"strings"
+	"unicode"
 )
 
 // Stanza or paragraph of Debian control file
@@ -157,6 +158,35 @@ func init() {
 	multilineFields["MD5Sum"] = true
 }
 
+func canonicalCase(field string) string {
+	upper := strings.ToUpper(field)
+	switch upper {
+	case "SHA1", "SHA256", "SHA512":
+		return upper
+	case "MD5SUM":
+		return "MD5Sum"
+	case "NOTAUTOMATIC":
+		return "NotAutomatic"
+	case "BUTAUTOMATICUPGRADES":
+		return "ButAutomaticUpgrades"
+	}
+
+	startOfWord := true
+
+	return strings.Map(func(r rune) rune {
+		if startOfWord {
+			startOfWord = false
+			return unicode.ToUpper(r)
+		}
+
+		if r == '-' {
+			startOfWord = true
+		}
+
+		return unicode.ToLower(r)
+	}, field)
+}
+
 // ControlFileReader implements reading of control files stanza by stanza
 type ControlFileReader struct {
 	scanner *bufio.Scanner
@@ -195,7 +225,7 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 			if len(parts) != 2 {
 				return nil, ErrMalformedStanza
 			}
-			lastField = parts[0]
+			lastField = canonicalCase(parts[0])
 			_, lastFieldMultiline = multilineFields[lastField]
 			if lastFieldMultiline {
 				stanza[lastField] = parts[1]

deb/format_test.go

@@ -123,6 +123,18 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 	c.Assert(strings.HasPrefix(str, "Package: "), Equals, true)
 }
 
+func (s *ControlFileSuite) TestCanonicalCase(c *C) {
+	c.Check(canonicalCase("Package"), Equals, "Package")
+	c.Check(canonicalCase("package"), Equals, "Package")
+	c.Check(canonicalCase("pAckaGe"), Equals, "Package")
+	c.Check(canonicalCase("MD5Sum"), Equals, "MD5Sum")
+	c.Check(canonicalCase("SHA1"), Equals, "SHA1")
+	c.Check(canonicalCase("SHA256"), Equals, "SHA256")
+	c.Check(canonicalCase("Package-List"), Equals, "Package-List")
+	c.Check(canonicalCase("package-list"), Equals, "Package-List")
+	c.Check(canonicalCase("packaGe-lIst"), Equals, "Package-List")
+}
+
 func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
 	for i := 0; i < c.N; i++ {
 		reader := bytes.NewBufferString(controlFile)

deb/import.go

@@ -91,6 +91,24 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
+		if p.Name == "" {
+			reporter.Warning("Empty package name on %s", file)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		if p.Version == "" {
+			reporter.Warning("Empty version on %s", file)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		if p.Architecture == "" {
+			reporter.Warning("Empty architecture on %s", file)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
 		var checksums utils.ChecksumInfo
 		checksums, err = utils.ChecksumsForFile(file)
 		if err != nil {

deb/publish.go

@@ -1000,7 +1000,8 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
 
 // Remove removes published repository, cleaning up directories, files
 func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly.PublishedStorageProvider,
-	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
+	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress,
+	force bool) error {
 	repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return err
@@ -1041,7 +1042,9 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 		err = collection.CleanupPrefixComponentFiles(repo.Prefix, cleanComponents,
 			publishedStorageProvider.GetPublishedStorage(storage), collectionFactory, progress)
 		if err != nil {
-			return err
+			if !force {
+				return fmt.Errorf("cleanup failed, use -force-drop to override: %s", err)
+			}
 		}
 	}
 

deb/publish_test.go

@@ -740,7 +740,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
-	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
@@ -760,10 +760,10 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 
-	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
 	c.Check(err, ErrorMatches, ".*not found")
 
-	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil)
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false)
 	c.Check(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
@@ -778,7 +778,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
-	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil)
+	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda")
@@ -800,7 +800,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) {
-	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil)
+	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")

deb/reflist.go

@@ -270,8 +270,9 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle
 
 // Merge merges reflist r into current reflist. If overrideMatching, merge
 // replaces matching packages (by architecture/name) with reference from r.
-// Otherwise, all packages are saved.
+// If ignoreConflicting is set, all packages are preserved, otherwise conflciting
-func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result *PackageRefList) {
+// packages are overwritten with packages from "right" snapshot.
+func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching, ignoreConflicting bool) (result *PackageRefList) {
 	var overriddenArch, overridenName []byte
 
 	// pointer to left and right reflists
@@ -314,7 +315,7 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result
 			partsR := bytes.Split(rr, []byte(" "))
 			archR, nameR, versionR := partsR[0][1:], partsR[1], partsR[2]
 
-			if bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) && bytes.Equal(versionL, versionR) {
+			if !ignoreConflicting && bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) && bytes.Equal(versionL, versionR) {
 				// conflicting duplicates with same arch, name, version, but different file hash
 				result.Refs = append(result.Refs, r.Refs[ir])
 				il++

deb/reflist_test.go

@@ -281,11 +281,11 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	reflistB := NewPackageRefListFromPackageList(listB)
 	reflistC := NewPackageRefListFromPackageList(listC)
 
-	mergeAB := reflistA.Merge(reflistB, true)
+	mergeAB := reflistA.Merge(reflistB, true, false)
-	mergeBA := reflistB.Merge(reflistA, true)
+	mergeBA := reflistB.Merge(reflistA, true, false)
-	mergeAC := reflistA.Merge(reflistC, true)
+	mergeAC := reflistA.Merge(reflistC, true, false)
-	mergeBC := reflistB.Merge(reflistC, true)
+	mergeBC := reflistB.Merge(reflistC, true, false)
-	mergeCB := reflistC.Merge(reflistB, true)
+	mergeCB := reflistC.Merge(reflistB, true, false)
 
 	c.Check(toStrSlice(mergeAB), DeepEquals,
 		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 dpkg 1.0 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
@@ -298,11 +298,11 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	c.Check(toStrSlice(mergeCB), DeepEquals,
 		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 dpkg 1.0 00000000", "Pi386 lib 1.0 00000000"})
 
-	mergeABall := reflistA.Merge(reflistB, false)
+	mergeABall := reflistA.Merge(reflistB, false, false)
-	mergeBAall := reflistB.Merge(reflistA, false)
+	mergeBAall := reflistB.Merge(reflistA, false, false)
-	mergeACall := reflistA.Merge(reflistC, false)
+	mergeACall := reflistA.Merge(reflistC, false, false)
-	mergeBCall := reflistB.Merge(reflistC, false)
+	mergeBCall := reflistB.Merge(reflistC, false, false)
-	mergeCBall := reflistC.Merge(reflistB, false)
+	mergeCBall := reflistC.Merge(reflistB, false, false)
 
 	c.Check(mergeABall, DeepEquals, mergeBAall)
 	c.Check(toStrSlice(mergeBAall), DeepEquals,
@@ -316,6 +316,11 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	c.Check(toStrSlice(mergeBCall), DeepEquals,
 		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445",
 			"Pi386 lib 1.0 00000000"})
+
+	mergeBCwithConflicts := reflistB.Merge(reflistC, false, true)
+	c.Check(toStrSlice(mergeBCwithConflicts), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044",
+			"Pi386 dpkg 1.0 00000000", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000"})
 }
 
 func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) {

man/aptly.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "APTLY" "1" "February 2015" "" ""
+.TH "APTLY" "1" "March 2015" "" ""
 .
 .SH "NAME"
 \fBaptly\fR \- Debian repository management tool
@@ -133,6 +133,10 @@ specifies paramaters for short PPA url expansion, if left blank they default to
 \fBS3PublishEndpoints\fR
 configuration of Amazon S3 publishing endpoints (see below)
 .
+.TP
+\fBSwiftPublishEndpoints\fR
+configuration of OpenStack Swift publishing endpoints (see below)
+.
 .SH "S3 PUBLISHING ENDPOINTS"
 aptly could be configured to publish repository directly to Amazon S3\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings:
 .
@@ -1082,6 +1086,13 @@ $ aptly publish drop wheezy
 .
 .IP "" 0
 .
+.P
+Options:
+.
+.TP
+\-\fBforce\-drop\fR=false
+remove published repository even if some files could not be cleaned up
+.
 .SH "LIST OF PUBLISHED REPOSITORIES"
 \fBaptly\fR \fBpublish\fR \fBlist\fR
 .
@@ -1469,6 +1480,17 @@ Example:
 .P
 $ aptly db cleanup
 .
+.P
+Options:
+.
+.TP
+\-\fBdry\-run\fR=false
+don\(cqt delete anything
+.
+.TP
+\-\fBverbose\fR=false
+be verbose when loading objects/removing them
+.
 .SH "RECOVER DB AFTER CRASH"
 \fBaptly\fR \fBdb\fR \fBrecover\fR
 .

system/t01_version/VersionTest_gold

@@ -1 +1 @@
-aptly version: 0.9
+aptly version: 0.9.5

system/t05_snapshot/SearchSnapshot5Test_gold

@@ -0,0 +1 @@
+ERROR: no results

system/t05_snapshot/search.py

@@ -37,3 +37,13 @@ class SearchSnapshot4Test(BaseTest):
     fixtureCmds = ["aptly snapshot create wheezy-main from mirror wheezy-main"]
     outputMatchPrepare = lambda _, s: "\n".join(sorted(s.split("\n")))
     runCmd = "aptly snapshot search -with-deps wheezy-main 'Name (nginx)'"
+
+
+class SearchSnapshot5Test(BaseTest):
+    """
+    search snapshot: no results
+    """
+    fixtureDB = True
+    fixtureCmds = ["aptly snapshot create wheezy-main from mirror wheezy-main"]
+    runCmd = "aptly snapshot search -with-deps wheezy-main 'Name (no-such-package)'"
+    expectedCode = 1

system/t08_db/CleanupDB10Test/pyspi_0.6.1-1.3.conflict.dsc

@@ -0,0 +1,12 @@
+Format: 1.0
+Source: pyspi
+Binary: python-at-spi
+Architecture: any
+Version: 0.6.1-1.3
+Maintainer: Jose Carlos Garcia Sogo <jsogo@debian.org>
+Homepage: http://people.redhat.com/zcerza/dogtail
+Standards-Version: 3.7.3
+Vcs-Svn: svn://svn.tribulaciones.org/srv/svn/pyspi/trunk
+Build-Depends: debhelper (>= 5), cdbs, libatspi-dev, python-pyrex, python-support (>= 0.4), python-all-dev, libx11-dev
+Files:
+ d41d8cd98f00b204e9800998ecf8427e 0 pyspi_0.6.1.orig.tar.gz

system/t08_db/CleanupDB10Test_gold

@@ -0,0 +1,7 @@
+Loading mirrors, local repos, snapshots and published repos...
+Loading list of all packages...
+Deleting unreferenced packages (0)...
+Building list of files referenced by packages...
+Building list of files in package pool...
+Deleting unreferenced files (0)...
+Compacting database...

system/t08_db/CleanupDB11Test_gold

@@ -0,0 +1,41 @@
+Loading mirrors, local repos, snapshots and published repos...
+Loading mirrors:
+- wheezy-backports
+- wheezy-updates-src
+- wheezy-main
+- wheezy-contrib
+- sensu
+- wheezy-main-src
+- wheezy-backports-src
+- wheezy-contrib-src
+- wheezy-non-free
+- wheezy-non-free-src
+- wheezy-updates
+Loading local repos:
+Loading snapshots:
+Loading published repositories:
+Loading list of all packages...
+Deleting unreferenced packages (7)...
+List of package keys to delete:
+ - Pall gnuplot 4.6.1-1~maverick2 36650cfe603d11a1
+ - Pall gnuplot-doc 4.6.1-1~maverick2 10c388f966074d29
+ - Pamd64 gnuplot-nox 4.6.1-1~maverick2 336b8733c4444003
+ - Pamd64 gnuplot-x11 4.6.1-1~maverick2 7300d8122b81b641
+ - Pi386 gnuplot-nox 4.6.1-1~maverick2 17785995cf0f815
+ - Pi386 gnuplot-x11 4.6.1-1~maverick2 d42e1d0d2f23740
+ - Psource gnuplot 4.6.1-1~maverick2 b8cd36358f5db41f
+Building list of files referenced by packages...
+Building list of files in package pool...
+Deleting unreferenced files (9)...
+List of files to be deleted:
+ - 02/1d/gnuplot_4.6.1-1~maverick2.dsc
+ - 10/32/gnuplot_4.6.1-1~maverick2.debian.tar.gz
+ - 17/ab/gnuplot-x11_4.6.1-1~maverick2_amd64.deb
+ - 25/a5/gnuplot-doc_4.6.1-1~maverick2_all.deb
+ - 49/12/gnuplot_4.6.1-1~maverick2_all.deb
+ - 4c/9a/gnuplot_4.6.1.orig.tar.gz
+ - a7/ef/gnuplot-nox_4.6.1-1~maverick2_i386.deb
+ - db/55/gnuplot-nox_4.6.1-1~maverick2_amd64.deb
+ - fc/ad/gnuplot-x11_4.6.1-1~maverick2_i386.deb
+Disk space freed: 10.85 MiB...
+Compacting database...

system/t08_db/CleanupDB12Test_gold

@@ -0,0 +1,42 @@
+Loading mirrors, local repos, snapshots and published repos...
+Loading mirrors:
+- wheezy-backports
+- wheezy-updates-src
+- wheezy-main
+- wheezy-contrib
+- sensu
+- wheezy-main-src
+- wheezy-backports-src
+- wheezy-contrib-src
+- wheezy-non-free
+- wheezy-non-free-src
+- wheezy-updates
+Loading local repos:
+Loading snapshots:
+Loading published repositories:
+Loading list of all packages...
+Deleting unreferenced packages (7)...
+List of package keys to delete:
+ - Pall gnuplot 4.6.1-1~maverick2 36650cfe603d11a1
+ - Pall gnuplot-doc 4.6.1-1~maverick2 10c388f966074d29
+ - Pamd64 gnuplot-nox 4.6.1-1~maverick2 336b8733c4444003
+ - Pamd64 gnuplot-x11 4.6.1-1~maverick2 7300d8122b81b641
+ - Pi386 gnuplot-nox 4.6.1-1~maverick2 17785995cf0f815
+ - Pi386 gnuplot-x11 4.6.1-1~maverick2 d42e1d0d2f23740
+ - Psource gnuplot 4.6.1-1~maverick2 b8cd36358f5db41f
+Skipped deletion, as -dry-run has been requested.
+Building list of files referenced by packages...
+Building list of files in package pool...
+Deleting unreferenced files (9)...
+List of files to be deleted:
+ - 02/1d/gnuplot_4.6.1-1~maverick2.dsc
+ - 10/32/gnuplot_4.6.1-1~maverick2.debian.tar.gz
+ - 17/ab/gnuplot-x11_4.6.1-1~maverick2_amd64.deb
+ - 25/a5/gnuplot-doc_4.6.1-1~maverick2_all.deb
+ - 49/12/gnuplot_4.6.1-1~maverick2_all.deb
+ - 4c/9a/gnuplot_4.6.1.orig.tar.gz
+ - a7/ef/gnuplot-nox_4.6.1-1~maverick2_i386.deb
+ - db/55/gnuplot-nox_4.6.1-1~maverick2_amd64.deb
+ - fc/ad/gnuplot-x11_4.6.1-1~maverick2_i386.deb
+Skipped file deletion, as -dry-run has been requested.
+Skipped DB compaction, as -dry-run has been requested.

system/t08_db/cleanup.py

@@ -111,3 +111,42 @@ class CleanupDB9Test(BaseTest):
     def check(self):
             self.check_output()
             self.check_cmd_output("aptly publish drop def", "publish_drop", match_prepare=self.expand_environ)
+
+
+class CleanupDB10Test(BaseTest):
+    """
+    cleanup db: conflict in packages, should not cleanup anything
+    """
+    fixtureCmds = [
+        "aptly repo create a",
+        "aptly repo create b",
+        "aptly repo add a ${files}",
+        "aptly repo add b ${testfiles}"
+    ]
+    runCmd = "aptly db cleanup"
+
+
+class CleanupDB11Test(BaseTest):
+    """
+    cleanup db: deleting packages and files, -verbose
+    """
+    fixtureDB = True
+    fixturePoolCopy = True
+    fixtureCmds = [
+        "aptly mirror drop gnuplot-maverick-src",
+        "aptly mirror drop gnuplot-maverick",
+    ]
+    runCmd = "aptly db cleanup -verbose"
+
+
+class CleanupDB12Test(BaseTest):
+    """
+    cleanup db: deleting packages and files, -verbose & -dry-run
+    """
+    fixtureDB = True
+    fixturePoolCopy = True
+    fixtureCmds = [
+        "aptly mirror drop gnuplot-maverick-src",
+        "aptly mirror drop gnuplot-maverick",
+    ]
+    runCmd = "aptly db cleanup -verbose -dry-run"

system/t09_repo/AddRepo15Test/pyspi_0.6-broken.dsc

@@ -0,0 +1,11 @@
+Format: 1.0
+sourcE: pyspi
+binary: python-at-spi
+Architecture: any
+Maintainer: Jose Carlos Garcia Sogo <jsogo@debian.org>
+Homepage: http://people.redhat.com/zcerza/dogtail
+Standards-Version: 3.7.3
+Vcs-svn: svn://svn.tribulaciones.org/srv/svn/pyspi/trunk
+Build-Depends: debhelper (>= 5), cdbs, libatspi-dev, python-pyrex, python-support (>= 0.4), python-all-dev, libx11-dev
+files:
+ d41d8cd98f00b204e9800998ecf8427e 0 pyspi_0.6.1.orig.tar.gz

system/t09_repo/AddRepo15Test/pyspi_0.6.1-1.3.conflict.dsc

@@ -0,0 +1,12 @@
+Format: 1.0
+sourcE: pyspi
+binary: python-at-spi
+Architecture: any
+Version: 0.6.1-1.3
+Maintainer: Jose Carlos Garcia Sogo <jsogo@debian.org>
+Homepage: http://people.redhat.com/zcerza/dogtail
+Standards-Version: 3.7.3
+Vcs-svn: svn://svn.tribulaciones.org/srv/svn/pyspi/trunk
+Build-Depends: debhelper (>= 5), cdbs, libatspi-dev, python-pyrex, python-support (>= 0.4), python-all-dev, libx11-dev
+files:
+ d41d8cd98f00b204e9800998ecf8427e 0 pyspi_0.6.1.orig.tar.gz

system/t09_repo/AddRepo15Test_gold

@@ -0,0 +1,6 @@
+Loading packages...
+[!] Empty version on /pyspi_0.6-broken.dsc
+[+] pyspi_0.6.1-1.3_source added
+[!] Some files were skipped due to errors:
+  /pyspi_0.6-broken.dsc
+ERROR: some files failed to be added

system/t09_repo/add.py

@@ -283,3 +283,15 @@ class AddRepo14Test(BaseTest):
         super(AddRepo14Test, self).check()
         # check pool
         self.check_file_not_empty('pool/00/35/libboost-program-options-dev_1.49.0.1_i386.deb')
+
+
+class AddRepo15Test(BaseTest):
+    """
+    add package with wrong case in stanza and missing fields
+    """
+    fixtureCmds = [
+        "aptly repo create -comment=Repo15 -distribution=squeeze repo15",
+    ]
+    runCmd = "aptly repo add repo15 ${testfiles}"
+    outputMatchPrepare = lambda self, s: s.replace(os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), self.__class__.__name__), "").replace(os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files"), "")
+    expectedCode = 1

system/t10_task/RunTask1Test_gold

@@ -21,6 +21,6 @@ End command output: ------------------------------
 4) [Running]: version
 
 Begin command output: ----------------------------
-aptly version: 0.9
+aptly version: 0.9.5
 
 End command output: ------------------------------

system/t12_api/version.py

@@ -7,4 +7,4 @@ class VersionAPITest(APITest):
     """
 
     def check(self):
-        self.check_equal(self.get("/api/version").json(), {'Version': '0.9'})
+        self.check_equal(self.get("/api/version").json(), {'Version': '0.9.5'})