Introduce back reflist merging without conflict removal. aptly db cleanup requires

full reference list collection. #217

Fixes bug with aptly db cleanup removing conflicting packages.

cmd/db_cleanup.go

@@ -27,7 +27,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			return err
 		}
 		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
 		}
 		return nil
 	})
@@ -41,7 +41,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			return err
 		}
 		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
 		}
 		return nil
 	})
@@ -54,7 +54,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		if err != nil {
 			return err
 		}
-		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false)
+		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
 		return nil
 	})
 	if err != nil {
@@ -71,7 +71,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		}
 
 		for _, component := range published.Components() {
-			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false)
+			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
 		}
 		return nil
 	})

cmd/snapshot_merge.go

@@ -39,7 +39,7 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 
 	result := sources[0].RefList()
 	for i := 1; i < len(sources); i++ {
-		result = result.Merge(sources[i].RefList(), overrideMatching)
+		result = result.Merge(sources[i].RefList(), overrideMatching, false)
 	}
 
 	if latest {

deb/reflist.go

@@ -270,8 +270,9 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle
 
 // Merge merges reflist r into current reflist. If overrideMatching, merge
 // replaces matching packages (by architecture/name) with reference from r.
-// Otherwise, all packages are saved.
+// If ignoreConflicting is set, all packages are preserved, otherwise conflciting
-func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result *PackageRefList) {
+// packages are overwritten with packages from "right" snapshot.
+func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching, ignoreConflicting bool) (result *PackageRefList) {
 	var overriddenArch, overridenName []byte
 
 	// pointer to left and right reflists
@@ -314,7 +315,7 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result
 			partsR := bytes.Split(rr, []byte(" "))
 			archR, nameR, versionR := partsR[0][1:], partsR[1], partsR[2]
 
-			if bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) && bytes.Equal(versionL, versionR) {
+			if !ignoreConflicting && bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) && bytes.Equal(versionL, versionR) {
 				// conflicting duplicates with same arch, name, version, but different file hash
 				result.Refs = append(result.Refs, r.Refs[ir])
 				il++

deb/reflist_test.go

@@ -281,11 +281,11 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	reflistB := NewPackageRefListFromPackageList(listB)
 	reflistC := NewPackageRefListFromPackageList(listC)
 
-	mergeAB := reflistA.Merge(reflistB, true)
+	mergeAB := reflistA.Merge(reflistB, true, false)
-	mergeBA := reflistB.Merge(reflistA, true)
+	mergeBA := reflistB.Merge(reflistA, true, false)
-	mergeAC := reflistA.Merge(reflistC, true)
+	mergeAC := reflistA.Merge(reflistC, true, false)
-	mergeBC := reflistB.Merge(reflistC, true)
+	mergeBC := reflistB.Merge(reflistC, true, false)
-	mergeCB := reflistC.Merge(reflistB, true)
+	mergeCB := reflistC.Merge(reflistB, true, false)
 
 	c.Check(toStrSlice(mergeAB), DeepEquals,
 		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 dpkg 1.0 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
@@ -298,11 +298,11 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	c.Check(toStrSlice(mergeCB), DeepEquals,
 		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 dpkg 1.0 00000000", "Pi386 lib 1.0 00000000"})
 
-	mergeABall := reflistA.Merge(reflistB, false)
+	mergeABall := reflistA.Merge(reflistB, false, false)
-	mergeBAall := reflistB.Merge(reflistA, false)
+	mergeBAall := reflistB.Merge(reflistA, false, false)
-	mergeACall := reflistA.Merge(reflistC, false)
+	mergeACall := reflistA.Merge(reflistC, false, false)
-	mergeBCall := reflistB.Merge(reflistC, false)
+	mergeBCall := reflistB.Merge(reflistC, false, false)
-	mergeCBall := reflistC.Merge(reflistB, false)
+	mergeCBall := reflistC.Merge(reflistB, false, false)
 
 	c.Check(mergeABall, DeepEquals, mergeBAall)
 	c.Check(toStrSlice(mergeBAall), DeepEquals,
@@ -316,6 +316,11 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	c.Check(toStrSlice(mergeBCall), DeepEquals,
 		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445",
 			"Pi386 lib 1.0 00000000"})
+
+	mergeBCwithConflicts := reflistB.Merge(reflistC, false, true)
+	c.Check(toStrSlice(mergeBCwithConflicts), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044",
+			"Pi386 dpkg 1.0 00000000", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000"})
 }
 
 func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) {

system/t08_db/CleanupDB10Test/pyspi_0.6.1-1.3.conflict.dsc

@@ -0,0 +1,12 @@
+Format: 1.0
+Source: pyspi
+Binary: python-at-spi
+Architecture: any
+Version: 0.6.1-1.3
+Maintainer: Jose Carlos Garcia Sogo <jsogo@debian.org>
+Homepage: http://people.redhat.com/zcerza/dogtail
+Standards-Version: 3.7.3
+Vcs-Svn: svn://svn.tribulaciones.org/srv/svn/pyspi/trunk
+Build-Depends: debhelper (>= 5), cdbs, libatspi-dev, python-pyrex, python-support (>= 0.4), python-all-dev, libx11-dev
+Files:
+ d41d8cd98f00b204e9800998ecf8427e 0 pyspi_0.6.1.orig.tar.gz

system/t08_db/CleanupDB10Test_gold

@@ -0,0 +1,7 @@
+Loading mirrors, local repos, snapshots and published repos...
+Loading list of all packages...
+Deleting unreferenced packages (0)...
+Building list of files referenced by packages...
+Building list of files in package pool...
+Deleting unreferenced files (0)...
+Compacting database...

system/t08_db/cleanup.py

@@ -111,3 +111,16 @@ class CleanupDB9Test(BaseTest):
     def check(self):
             self.check_output()
             self.check_cmd_output("aptly publish drop def", "publish_drop", match_prepare=self.expand_environ)
+
+
+class CleanupDB10Test(BaseTest):
+    """
+    cleanup db: conflict in packages, should not cleanup anything
+    """
+    fixtureCmds = [
+        "aptly repo create a",
+        "aptly repo create b",
+        "aptly repo add a ${files}",
+        "aptly repo add b ${testfiles}"
+    ]
+    runCmd = "aptly db cleanup"