Merge branch 'f-newestpkg' of https://github.com/ryanuber/aptly into ryanuber-lastest-merge

cmd/snapshot_merge.go

@@ -28,10 +28,17 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 		}
 	}
 
+	latest := context.flags.Lookup("latest").Value.Get().(bool)
+	overrideMatching := !latest
+
 	result := sources[0].RefList()
 
 	for i := 1; i < len(sources); i++ {
-		result = result.Merge(sources[i].RefList(), true)
+		result = result.Merge(sources[i].RefList(), overrideMatching)
+	}
+
+	if latest {
+		deb.FilterLatestRefs(result)
 	}
 
 	sourceDescription := make([]string, len(sources))
@@ -71,5 +78,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("latest", false, "Use only the latest version of each package")
+
 	return cmd
 }

deb/reflist.go

@@ -219,8 +219,9 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle
 	return
 }
 
-// Merge merges reflist r into current reflist. If overrideMatching, merge replaces matching packages (by architecture/name)
+// Merge merges reflist r into current reflist. If overrideMatching, merge
-// with reference from r, otherwise all packages are saved.
+// replaces matching packages (by architecture/name) with reference from r.
+// Otherwise, all packages are saved.
 func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result *PackageRefList) {
 	// pointer to left and right reflists
 	il, ir := 0, 0
@@ -278,9 +279,48 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result
 				result.Refs = append(result.Refs, r.Refs[ir])
 				ir++
 			}
-
 		}
 	}
 
 	return
 }
+
+// FilterLatestRefs takes in a reflist with potentially multiples of the same
+// packages and reduces it to only the latest of each package. The operations
+// are done in-place. This implements a "latest wins" approach which can be used
+// while merging two or more snapshots together.
+func FilterLatestRefs(r *PackageRefList) {
+	// A running tab of latest seen package refs.
+	latestRefs := make(map[string]int)
+
+	for i := 0; i < len(r.Refs); i++ {
+		partsL := bytes.Split(r.Refs[i], []byte(" "))
+		archL, nameL, verL := partsL[0][1:], partsL[1], partsL[2]
+		pkgId := string(nameL) + "." + string(archL)
+
+		// If the package hasn't been seen before, add it and advance.
+		if _, ok := latestRefs[pkgId]; !ok {
+			latestRefs[pkgId] = i
+			continue
+		}
+
+		// If we've already seen this package, check versions
+		partsR := bytes.Split(r.Refs[latestRefs[pkgId]], []byte(" "))
+		verR := partsR[2]
+		vres := CompareVersions(string(verL), string(verR))
+
+		// Remove the older or duplicate refs from the result
+		if vres > 0 {
+			old := latestRefs[pkgId]
+			r.Refs = append(r.Refs[0:old], r.Refs[old+1:]...)
+			latestRefs[pkgId] = i - 1
+		} else {
+			r.Refs = append(r.Refs[0:i], r.Refs[i+1:]...)
+		}
+
+		// Compensate for the reduced set
+		i -= 1
+	}
+
+	return
+}

deb/reflist_test.go

@@ -14,6 +14,14 @@ type PackageRefListSuite struct {
 
 var _ = Suite(&PackageRefListSuite{})
 
+func toStrSlice(reflist *PackageRefList) (result []string) {
+	result = make([]string, reflist.Len())
+	for i, r := range reflist.Refs {
+		result[i] = string(r)
+	}
+	return
+}
+
 func (s *PackageRefListSuite) SetUpTest(c *C) {
 	s.list = NewPackageList()
 
@@ -250,14 +258,6 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	reflistA := NewPackageRefListFromPackageList(listA)
 	reflistB := NewPackageRefListFromPackageList(listB)
 
-	toStrSlice := func(reflist *PackageRefList) (result []string) {
-		result = make([]string, reflist.Len())
-		for i, r := range reflist.Refs {
-			result[i] = string(r)
-		}
-		return
-	}
-
 	mergeAB := reflistA.Merge(reflistB, true)
 	mergeBA := reflistB.Merge(reflistA, true)
 
@@ -273,3 +273,40 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	c.Check(toStrSlice(mergeBAall), DeepEquals,
 		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"})
 }
+
+func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) {
+	packages := []*Package{
+		&Package{Name: "lib", Version: "1.0", Architecture: "i386"},
+		&Package{Name: "lib", Version: "1.1", Architecture: "i386"},
+		&Package{Name: "lib", Version: "1.2", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.2", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.3", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.4", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.5", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.6", Architecture: "i386"},
+	}
+
+	rl := NewPackageList()
+	rl.Add(packages[0])
+	rl.Add(packages[1])
+	rl.Add(packages[2])
+	rl.Add(packages[3])
+	rl.Add(packages[7])
+	rl.Add(packages[0])
+	rl.Add(packages[2])
+	rl.Add(packages[4])
+	rl.Add(packages[5])
+	rl.Add(packages[6])
+	rl.Add(packages[3])
+	rl.Add(packages[3])
+	rl.Add(packages[4])
+	rl.Add(packages[4])
+	rl.Add(packages[7])
+	rl.Add(packages[7])
+
+	result := NewPackageRefListFromPackageList(rl)
+	FilterLatestRefs(result)
+
+	c.Check(toStrSlice(result), DeepEquals,
+		[]string{"Pi386 dpkg 1.6", "Pi386 lib 1.2"})
+}

man/aptly.1

@@ -640,7 +640,7 @@ display diff only for matching packages (don\(cqt display missing packages)
 \fBaptly\fR \fBsnapshot\fR \fBmerge\fR \fIdestination\fR \fIsource\fR [\fIsource\fR\|\.\|\.\|\.]
 .
 .P
-Merge command merges several \fIsource\fR snapshots into one \fIdestination\fR snapshot\. Merge happens from left to right\. Packages with the same name\-architecture pair are replaced during merge (package from latest snapshot on the list wins)\. If run with only one source snapshot, merge copies \fIsource\fR into \fIdestination\fR\.
+Merge command merges several \fIsource\fR snapshots into one \fIdestination\fR snapshot\. Merge happens from left to right\. By default, packages with the same name\-architecture pair are replaced during merge (package from latest snapshot on the list wins)\. If run with only one source snapshot, merge copies \fIsource\fR into \fIdestination\fR\.
 .
 .P
 Example:
@@ -655,6 +655,13 @@ $ aptly snapshot merge wheezy\-w\-backports wheezy\-main wheezy\-backports
 .
 .IP "" 0
 .
+.P
+Options:
+.
+.TP
+\-\fBlatest\fR=false
+use only the latest version of each package in the merged snapshot
+.
 .SH "DELETE SNAPSHOT"
 \fBaptly\fR \fBsnapshot\fR \fBdrop\fR \fIname\fR
 .

system/t05_snapshot/MergeSnapshot6Test_gold

@@ -0,0 +1,3 @@
+
+Snapshot snap4 successfully created.
+You can run 'aptly publish snapshot snap4' to publish snapshot as Debian repository.

system/t05_snapshot/merge.py

@@ -78,3 +78,17 @@ class MergeSnapshot5Test(BaseTest):
     ]
     runCmd = "aptly snapshot merge snap1 snap1"
     expectedCode = 1
+
+
+class MergeSnapshot6Test(BaseTest):
+    """
+    merge snapshots: use latest versions only
+    """
+    fixtureDB = True
+    fixtureCmds = [
+        "aptly snapshot create snap1 from mirror wheezy-main",
+        "aptly snapshot create snap2 from mirror wheezy-non-free",
+        "aptly snapshot create snap3 from mirror wheezy-backports",
+    ]
+    runCmd = "aptly snapshot merge -latest snap4 snap1 snap2 snap3"
+    expectedCode = 0