From 992a5cee3776d031fa58679dc9ee465a3fdfde09 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Mon, 21 Apr 2014 18:45:06 -0700 Subject: [PATCH 01/16] snapshot: first pass at newest-wins functionality. --- cmd/db_cleanup.go | 6 +++--- cmd/snapshot_merge.go | 2 +- deb/reflist.go | 25 ++++++++++++++++++++++--- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/cmd/db_cleanup.go b/cmd/db_cleanup.go index 18d1dda7..cb645d0c 100644 --- a/cmd/db_cleanup.go +++ b/cmd/db_cleanup.go @@ -27,7 +27,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error { return err } if repo.RefList() != nil { - existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false) + existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, false) } return nil }) @@ -41,7 +41,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error { return err } if repo.RefList() != nil { - existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false) + existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, false) } return nil }) @@ -54,7 +54,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error { if err != nil { return err } - existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false) + existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, false) return nil }) if err != nil { diff --git a/cmd/snapshot_merge.go b/cmd/snapshot_merge.go index 653be35e..a84da360 100644 --- a/cmd/snapshot_merge.go +++ b/cmd/snapshot_merge.go @@ -31,7 +31,7 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error { result := sources[0].RefList() for i := 1; i < len(sources); i++ { - result = result.Merge(sources[i].RefList(), true) + result = result.Merge(sources[i].RefList(), true, true) } sourceDescription := make([]string, len(sources)) diff --git a/deb/reflist.go b/deb/reflist.go index fe37813d..3e810fd5 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -219,9 +219,13 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle return } -// Merge merges reflist r into current reflist. If overrideMatching, merge replaces matching packages (by architecture/name) -// with reference from r, otherwise all packages are saved. -func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result *PackageRefList) { +// Merge merges reflist r into current reflist. If overrideMatching, merge +// replaces matching packages (by architecture/name) with reference from r. If +// newestWins, compare versions between common packages and take the latest from +// the set. Otherwise, all packages are saved. +func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, + newestWins bool) (result *PackageRefList) { + // pointer to left and right reflists il, ir := 0, 0 // length of reflists @@ -269,6 +273,21 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result continue } } + if newestWins { + partsL := bytes.Split(rl, []byte(" ")) + verL := string(partsL[2]) + + partsR := bytes.Split(rr, []byte(" ")) + verR := string(partsR[2]) + + vres := CompareVersions(verL, verR) + if vres <= 0 { + result.Refs = append(result.Refs, r.Refs[ir]) + il++ + ir++ + continue + } + } // otherwise append smallest of two if rel < 0 { From cd369f5fa0c415b87730c1e4882c55099127cf58 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Mon, 21 Apr 2014 19:38:58 -0700 Subject: [PATCH 02/16] snapshot: add cli flag for taking newest during merge --- cmd/snapshot_merge.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/snapshot_merge.go b/cmd/snapshot_merge.go index a84da360..ba06f521 100644 --- a/cmd/snapshot_merge.go +++ b/cmd/snapshot_merge.go @@ -28,10 +28,16 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error { } } + newest := context.flags.Lookup("newest").Value.Get().(bool) + result := sources[0].RefList() for i := 1; i < len(sources); i++ { - result = result.Merge(sources[i].RefList(), true, true) + if newest { + result = result.Merge(sources[i].RefList(), false, true) + } else { + result = result.Merge(sources[i].RefList(), true, false) + } } sourceDescription := make([]string, len(sources)) @@ -71,5 +77,7 @@ Example: `, } + cmd.Flag.Bool("newest", false, "Take newest package of set during merge") + return cmd } From dbcfd6f58bd735c7e18a62dbc9b60257968139ed Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Tue, 22 Apr 2014 00:29:12 -0700 Subject: [PATCH 03/16] snapshot: keep a tab of seen packages and only include the latest copy during merge --- deb/reflist.go | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/deb/reflist.go b/deb/reflist.go index 3e810fd5..ba5731ce 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -226,6 +226,10 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, newestWins bool) (result *PackageRefList) { + // A running tab of packages observed during a merge. Used when -newest is + // passed to make sure only the newest version is carried into the snapshot. + var seen []string + // pointer to left and right reflists il, ir := 0, 0 // length of reflists @@ -234,6 +238,7 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, result = &PackageRefList{} result.Refs = make([][]byte, 0, ll+lr) +OUTER: // until we reached end of both lists for il < ll || ir < lr { // if we've exhausted left list, pull the rest from the right @@ -275,12 +280,25 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, } if newestWins { partsL := bytes.Split(rl, []byte(" ")) - verL := string(partsL[2]) + nameL, archL, verL := partsL[0][1:], partsL[1], partsL[2] + pkgL := string(nameL) + "." + string(archL) partsR := bytes.Split(rr, []byte(" ")) - verR := string(partsR[2]) + verR := partsR[2] - vres := CompareVersions(verL, verR) + // If we've already seen this package, regardless of version, + // just skip it. + for _, s := range seen { + if s == pkgL { + il++ + ir++ + continue OUTER + } + } + + seen = append(seen, pkgL) + + vres := CompareVersions(string(verL), string(verR)) if vres <= 0 { result.Refs = append(result.Refs, r.Refs[ir]) il++ @@ -297,7 +315,6 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, result.Refs = append(result.Refs, r.Refs[ir]) ir++ } - } } From 60d48e890c40c92c0ee8eb1016288b388f859002 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Tue, 22 Apr 2014 01:47:35 -0700 Subject: [PATCH 04/16] snapshot: Move 'newest' logic out of main compilation loop and evaluate the reflist afterward. --- deb/reflist.go | 65 +++++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/deb/reflist.go b/deb/reflist.go index ba5731ce..631c3b6a 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -226,10 +226,6 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, newestWins bool) (result *PackageRefList) { - // A running tab of packages observed during a merge. Used when -newest is - // passed to make sure only the newest version is carried into the snapshot. - var seen []string - // pointer to left and right reflists il, ir := 0, 0 // length of reflists @@ -238,7 +234,6 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, result = &PackageRefList{} result.Refs = make([][]byte, 0, ll+lr) -OUTER: // until we reached end of both lists for il < ll || ir < lr { // if we've exhausted left list, pull the rest from the right @@ -278,34 +273,6 @@ OUTER: continue } } - if newestWins { - partsL := bytes.Split(rl, []byte(" ")) - nameL, archL, verL := partsL[0][1:], partsL[1], partsL[2] - pkgL := string(nameL) + "." + string(archL) - - partsR := bytes.Split(rr, []byte(" ")) - verR := partsR[2] - - // If we've already seen this package, regardless of version, - // just skip it. - for _, s := range seen { - if s == pkgL { - il++ - ir++ - continue OUTER - } - } - - seen = append(seen, pkgL) - - vres := CompareVersions(string(verL), string(verR)) - if vres <= 0 { - result.Refs = append(result.Refs, r.Refs[ir]) - il++ - ir++ - continue - } - } // otherwise append smallest of two if rel < 0 { @@ -318,5 +285,37 @@ OUTER: } } + if newestWins { + // A running tab of packages observed during a merge. Used when -newest + // is passed to ensure the newest version is carried into the snapshot. + refs := make(map[string][]byte) + + OUTER: + for _, ref := range result.Refs { + partsL := bytes.Split(ref, []byte(" ")) + nameL, archL, verL := partsL[0][1:], partsL[1], partsL[2] + pkgL := string(nameL) + "." + string(archL) + + // If we've already seen this package, regardless of version, + // just skip it. + if _, ok := refs[pkgL]; ok { + vres := CompareVersions(string(verL), string(refs[pkgL])) + if vres <= 0 { + refs[pkgL] = ref + il++ + ir++ + continue OUTER + } + } + + refs[pkgL] = ref + } + + result.Refs = [][]byte{} + for _, ref := range refs { + result.Refs = append(result.Refs, ref) + } + } + return } From 133d67bffa04f7af8f9f85b2a26914a3eec39be1 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Tue, 22 Apr 2014 01:51:33 -0700 Subject: [PATCH 05/16] snapshot: newest -> latest --- cmd/snapshot_merge.go | 6 +++--- deb/reflist.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/snapshot_merge.go b/cmd/snapshot_merge.go index ba06f521..e94f731e 100644 --- a/cmd/snapshot_merge.go +++ b/cmd/snapshot_merge.go @@ -28,12 +28,12 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error { } } - newest := context.flags.Lookup("newest").Value.Get().(bool) + latest := context.flags.Lookup("latest").Value.Get().(bool) result := sources[0].RefList() for i := 1; i < len(sources); i++ { - if newest { + if latest { result = result.Merge(sources[i].RefList(), false, true) } else { result = result.Merge(sources[i].RefList(), true, false) @@ -77,7 +77,7 @@ Example: `, } - cmd.Flag.Bool("newest", false, "Take newest package of set during merge") + cmd.Flag.Bool("latest", false, "Use only the latest version of all packages") return cmd } diff --git a/deb/reflist.go b/deb/reflist.go index 631c3b6a..02845214 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -296,8 +296,8 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, nameL, archL, verL := partsL[0][1:], partsL[1], partsL[2] pkgL := string(nameL) + "." + string(archL) - // If we've already seen this package, regardless of version, - // just skip it. + // If we've already seen this package, check if this version is + // newer. If it is, replace the older ref. if _, ok := refs[pkgL]; ok { vres := CompareVersions(string(verL), string(refs[pkgL])) if vres <= 0 { From a93052aa8a569b38d8c4d5c69c397b8a20d53bce Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Tue, 22 Apr 2014 10:58:20 -0700 Subject: [PATCH 06/16] snapshot: merge adjustments for -latest flag --- deb/reflist.go | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/deb/reflist.go b/deb/reflist.go index 02845214..15c25207 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -286,33 +286,37 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, } if newestWins { - // A running tab of packages observed during a merge. Used when -newest + // A running tab of packages observed during a merge. Used when -latest // is passed to ensure the newest version is carried into the snapshot. - refs := make(map[string][]byte) + latestRefs := make(map[string][]byte) - OUTER: for _, ref := range result.Refs { partsL := bytes.Split(ref, []byte(" ")) - nameL, archL, verL := partsL[0][1:], partsL[1], partsL[2] - pkgL := string(nameL) + "." + string(archL) + archL, nameL, verL := partsL[0][1:], partsL[1], partsL[2] + pkgId := string(nameL) + "." + string(archL) + + // If the package hasn't been seen before, just add it. + if _, ok := latestRefs[pkgId]; !ok { + latestRefs[pkgId] = ref + il++ + ir++ + continue + } // If we've already seen this package, check if this version is // newer. If it is, replace the older ref. - if _, ok := refs[pkgL]; ok { - vres := CompareVersions(string(verL), string(refs[pkgL])) - if vres <= 0 { - refs[pkgL] = ref - il++ - ir++ - continue OUTER - } - } + partsR := bytes.Split(latestRefs[pkgId], []byte(" ")) + verR := partsR[2] - refs[pkgL] = ref + vres := CompareVersions(string(verL), string(verR)) + if vres > 0 { + latestRefs[pkgId] = ref + } } + // Replace the result with the merged copy of latest-only refs. result.Refs = [][]byte{} - for _, ref := range refs { + for _, ref := range latestRefs { result.Refs = append(result.Refs, ref) } } From 1a735e849bbd009718f0efaacd88e90e16108ec0 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Tue, 22 Apr 2014 14:08:17 -0700 Subject: [PATCH 07/16] snapshot: alter result generation in Merge() to accommodate snapshot diff --- deb/reflist.go | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/deb/reflist.go b/deb/reflist.go index 15c25207..f9c28838 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -286,38 +286,43 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, } if newestWins { - // A running tab of packages observed during a merge. Used when -latest - // is passed to ensure the newest version is carried into the snapshot. - latestRefs := make(map[string][]byte) + // A running tab of package references we want to keep. Only the latest + // package reference is kept. + latestRefs := make(map[string]int) - for _, ref := range result.Refs { + for i, ref := range result.Refs { partsL := bytes.Split(ref, []byte(" ")) archL, nameL, verL := partsL[0][1:], partsL[1], partsL[2] pkgId := string(nameL) + "." + string(archL) // If the package hasn't been seen before, just add it. if _, ok := latestRefs[pkgId]; !ok { - latestRefs[pkgId] = ref - il++ - ir++ + latestRefs[pkgId] = i continue } // If we've already seen this package, check if this version is // newer. If it is, replace the older ref. - partsR := bytes.Split(latestRefs[pkgId], []byte(" ")) + partsR := bytes.Split(result.Refs[latestRefs[pkgId]], []byte(" ")) verR := partsR[2] vres := CompareVersions(string(verL), string(verR)) if vres > 0 { - latestRefs[pkgId] = ref + latestRefs[pkgId] = i } } - // Replace the result with the merged copy of latest-only refs. - result.Refs = [][]byte{} - for _, ref := range latestRefs { - result.Refs = append(result.Refs, ref) + offset := 0 + OUTER: + for i, _ := range result.Refs { + for _, ki := range latestRefs { + if ki == i { + continue OUTER + } + } + idx := i - offset + result.Refs = append(result.Refs[0:idx], result.Refs[idx+1:]...) + offset++ } } From 89eafd1b2174e9b5bad059b8cd741270e2838cd4 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Tue, 22 Apr 2014 15:28:03 -0700 Subject: [PATCH 08/16] Initial pass at testing merged snapshotting with -latest flag --- deb/reflist_test.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/deb/reflist_test.go b/deb/reflist_test.go index e376eb9e..7393fa9e 100644 --- a/deb/reflist_test.go +++ b/deb/reflist_test.go @@ -247,8 +247,17 @@ func (s *PackageRefListSuite) TestMerge(c *C) { listB.Add(packages[5]) listB.Add(packages[6]) + listC := NewPackageList() + listC.Add(packages[3]) + listC.Add(packages[3]) + listC.Add(packages[4]) + listC.Add(packages[4]) + listC.Add(packages[7]) + listC.Add(packages[7]) + reflistA := NewPackageRefListFromPackageList(listA) reflistB := NewPackageRefListFromPackageList(listB) + reflistC := NewPackageRefListFromPackageList(listC) toStrSlice := func(reflist *PackageRefList) (result []string) { result = make([]string, reflist.Len()) @@ -258,18 +267,29 @@ func (s *PackageRefListSuite) TestMerge(c *C) { return } - mergeAB := reflistA.Merge(reflistB, true) - mergeBA := reflistB.Merge(reflistA, true) + mergeAB := reflistA.Merge(reflistB, true, false) + mergeBA := reflistB.Merge(reflistA, true, false) c.Check(toStrSlice(mergeAB), DeepEquals, []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 lib 1.0", "Psparc xyz 1.0"}) c.Check(toStrSlice(mergeBA), DeepEquals, []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"}) - mergeABall := reflistA.Merge(reflistB, false) - mergeBAall := reflistB.Merge(reflistA, false) + mergeABall := reflistA.Merge(reflistB, false, false) + mergeBAall := reflistB.Merge(reflistA, false, false) c.Check(mergeABall, DeepEquals, mergeBAall) c.Check(toStrSlice(mergeBAall), DeepEquals, []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"}) + + mergeABClatest := reflistA.Merge(reflistB, false, true) + mergeABClatest = mergeABClatest.Merge(reflistC, false, true) + + mergeCBAlatest := reflistC.Merge(reflistB, false, true) + mergeCBAlatest = mergeCBAlatest.Merge(reflistA, false, true) + + c.Check(toStrSlice(mergeABClatest), DeepEquals, + []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"}) + c.Check(toStrSlice(mergeCBAlatest), DeepEquals, + []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"}) } From 0d8debe7b69f795179cf211adabafe0aa888b19e Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Tue, 22 Apr 2014 17:11:24 -0700 Subject: [PATCH 09/16] snapshot: simplify merging latest packages --- deb/reflist.go | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/deb/reflist.go b/deb/reflist.go index f9c28838..3155797d 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -290,39 +290,32 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, // package reference is kept. latestRefs := make(map[string]int) - for i, ref := range result.Refs { - partsL := bytes.Split(ref, []byte(" ")) + i := 0 + for _ = range result.Refs { + partsL := bytes.Split(result.Refs[i], []byte(" ")) archL, nameL, verL := partsL[0][1:], partsL[1], partsL[2] pkgId := string(nameL) + "." + string(archL) - // If the package hasn't been seen before, just add it. + // If the package hasn't been seen before, add it and advance. if _, ok := latestRefs[pkgId]; !ok { latestRefs[pkgId] = i + i++ continue } - // If we've already seen this package, check if this version is - // newer. If it is, replace the older ref. + // If we've already seen this package, check versions partsR := bytes.Split(result.Refs[latestRefs[pkgId]], []byte(" ")) verR := partsR[2] - vres := CompareVersions(string(verL), string(verR)) - if vres > 0 { - latestRefs[pkgId] = i - } - } - offset := 0 - OUTER: - for i, _ := range result.Refs { - for _, ki := range latestRefs { - if ki == i { - continue OUTER - } + // Remove the older or duplicate refs from the result + if vres <= 0 { + result.Refs = append(result.Refs[0:i], result.Refs[i+1:]...) + latestRefs[pkgId] = i + } else { + oi := latestRefs[pkgId] + result.Refs = append(result.Refs[0:oi], result.Refs[oi+1:]...) } - idx := i - offset - result.Refs = append(result.Refs[0:idx], result.Refs[idx+1:]...) - offset++ } } From e6992d822dd453bcb11a1b2d1aac5c1a89d2af46 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Tue, 22 Apr 2014 17:45:43 -0700 Subject: [PATCH 10/16] Updated man page for snapshot merging with -latest flag --- man/aptly.1 | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/man/aptly.1 b/man/aptly.1 index 1c092e5e..ffb0c852 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -640,7 +640,7 @@ display diff only for matching packages (don\(cqt display missing packages) \fBaptly\fR \fBsnapshot\fR \fBmerge\fR \fIdestination\fR \fIsource\fR [\fIsource\fR\|\.\|\.\|\.] . .P -Merge command merges several \fIsource\fR snapshots into one \fIdestination\fR snapshot\. Merge happens from left to right\. Packages with the same name\-architecture pair are replaced during merge (package from latest snapshot on the list wins)\. If run with only one source snapshot, merge copies \fIsource\fR into \fIdestination\fR\. +Merge command merges several \fIsource\fR snapshots into one \fIdestination\fR snapshot\. Merge happens from left to right\. By default, packages with the same name\-architecture pair are replaced during merge (package from latest snapshot on the list wins)\. If run with only one source snapshot, merge copies \fIsource\fR into \fIdestination\fR\. . .P Example: @@ -655,6 +655,13 @@ $ aptly snapshot merge wheezy\-w\-backports wheezy\-main wheezy\-backports . .IP "" 0 . +.P +Options: +. +.TP +\-\fBlatest\fR=false +use only the latest version of each package in the merged snapshot +. .SH "DELETE SNAPSHOT" \fBaptly\fR \fBsnapshot\fR \fBdrop\fR \fIname\fR . From d1cc562f3ccd8163dd81d2cdb3fe73b11f4ed125 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Tue, 22 Apr 2014 17:48:53 -0700 Subject: [PATCH 11/16] cmd/snapshot_merge: reword -latest flag to match man page --- cmd/snapshot_merge.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/snapshot_merge.go b/cmd/snapshot_merge.go index e94f731e..c601924a 100644 --- a/cmd/snapshot_merge.go +++ b/cmd/snapshot_merge.go @@ -77,7 +77,7 @@ Example: `, } - cmd.Flag.Bool("latest", false, "Use only the latest version of all packages") + cmd.Flag.Bool("latest", false, "Use only the latest version of each package") return cmd } From d9f867328666627dc3a719fb1f6e7b284aa04513 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Tue, 22 Apr 2014 19:15:34 -0700 Subject: [PATCH 12/16] snapshot: break out FilterLatestPackages to its own function --- deb/reflist.go | 78 ++++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/deb/reflist.go b/deb/reflist.go index 3155797d..f8cf5731 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -221,10 +221,10 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle // Merge merges reflist r into current reflist. If overrideMatching, merge // replaces matching packages (by architecture/name) with reference from r. If -// newestWins, compare versions between common packages and take the latest from +// latestWins, compare versions between common packages and take the latest from // the set. Otherwise, all packages are saved. func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, - newestWins bool) (result *PackageRefList) { + latestWins bool) (result *PackageRefList) { // pointer to left and right reflists il, ir := 0, 0 @@ -285,39 +285,49 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, } } - if newestWins { - // A running tab of package references we want to keep. Only the latest - // package reference is kept. - latestRefs := make(map[string]int) - - i := 0 - for _ = range result.Refs { - partsL := bytes.Split(result.Refs[i], []byte(" ")) - archL, nameL, verL := partsL[0][1:], partsL[1], partsL[2] - pkgId := string(nameL) + "." + string(archL) - - // If the package hasn't been seen before, add it and advance. - if _, ok := latestRefs[pkgId]; !ok { - latestRefs[pkgId] = i - i++ - continue - } - - // If we've already seen this package, check versions - partsR := bytes.Split(result.Refs[latestRefs[pkgId]], []byte(" ")) - verR := partsR[2] - vres := CompareVersions(string(verL), string(verR)) - - // Remove the older or duplicate refs from the result - if vres <= 0 { - result.Refs = append(result.Refs[0:i], result.Refs[i+1:]...) - latestRefs[pkgId] = i - } else { - oi := latestRefs[pkgId] - result.Refs = append(result.Refs[0:oi], result.Refs[oi+1:]...) - } - } + // Filter results down to the latest packages only if requested + if latestWins { + result = FilterLatestPackages(result) } return } + +// FilterLatestPackages takes in a reflist with potentially multiples of the +// same packages and returns a reflist containing only the latest of each +// package. This implements a "latest wins" approach which can be used while +// merging two or more snapshots together. +func FilterLatestPackages(r *PackageRefList) *PackageRefList { + // A running tab of latest seen package refs. + latestRefs := make(map[string]int) + + i := 0 + for _ = range r.Refs { + partsL := bytes.Split(r.Refs[i], []byte(" ")) + archL, nameL, verL := partsL[0][1:], partsL[1], partsL[2] + pkgId := string(nameL) + "." + string(archL) + + // If the package hasn't been seen before, add it and advance. + if _, ok := latestRefs[pkgId]; !ok { + latestRefs[pkgId] = i + i++ + continue + } + + // If we've already seen this package, check versions + partsR := bytes.Split(r.Refs[latestRefs[pkgId]], []byte(" ")) + verR := partsR[2] + vres := CompareVersions(string(verL), string(verR)) + + // Remove the older or duplicate refs from the result + if vres <= 0 { + r.Refs = append(r.Refs[0:i], r.Refs[i+1:]...) + latestRefs[pkgId] = i + } else { + oi := latestRefs[pkgId] + r.Refs = append(r.Refs[0:oi], r.Refs[oi+1:]...) + } + } + + return r +} From 385ac1afd0ca0f660ecd68a5fcb17b280cc55c37 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Tue, 22 Apr 2014 22:21:38 -0700 Subject: [PATCH 13/16] snapshot: explicity call FilterLatestRefs() where needed rather than calling from Merge() --- cmd/db_cleanup.go | 6 +++--- cmd/snapshot_merge.go | 11 ++++++----- deb/reflist.go | 41 +++++++++++++++++------------------------ 3 files changed, 26 insertions(+), 32 deletions(-) diff --git a/cmd/db_cleanup.go b/cmd/db_cleanup.go index cb645d0c..18d1dda7 100644 --- a/cmd/db_cleanup.go +++ b/cmd/db_cleanup.go @@ -27,7 +27,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error { return err } if repo.RefList() != nil { - existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, false) + existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false) } return nil }) @@ -41,7 +41,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error { return err } if repo.RefList() != nil { - existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, false) + existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false) } return nil }) @@ -54,7 +54,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error { if err != nil { return err } - existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, false) + existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false) return nil }) if err != nil { diff --git a/cmd/snapshot_merge.go b/cmd/snapshot_merge.go index c601924a..b7e85656 100644 --- a/cmd/snapshot_merge.go +++ b/cmd/snapshot_merge.go @@ -29,15 +29,16 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error { } latest := context.flags.Lookup("latest").Value.Get().(bool) + overrideMatching := !latest result := sources[0].RefList() for i := 1; i < len(sources); i++ { - if latest { - result = result.Merge(sources[i].RefList(), false, true) - } else { - result = result.Merge(sources[i].RefList(), true, false) - } + result = result.Merge(sources[i].RefList(), overrideMatching) + } + + if latest { + result = deb.FilterLatestRefs(result) } sourceDescription := make([]string, len(sources)) diff --git a/deb/reflist.go b/deb/reflist.go index f8cf5731..a6eb85cf 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -220,12 +220,9 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle } // Merge merges reflist r into current reflist. If overrideMatching, merge -// replaces matching packages (by architecture/name) with reference from r. If -// latestWins, compare versions between common packages and take the latest from -// the set. Otherwise, all packages are saved. -func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, - latestWins bool) (result *PackageRefList) { - +// replaces matching packages (by architecture/name) with reference from r. +// Otherwise, all packages are saved. +func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result *PackageRefList) { // pointer to left and right reflists il, ir := 0, 0 // length of reflists @@ -285,24 +282,18 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool, } } - // Filter results down to the latest packages only if requested - if latestWins { - result = FilterLatestPackages(result) - } - return } -// FilterLatestPackages takes in a reflist with potentially multiples of the -// same packages and returns a reflist containing only the latest of each -// package. This implements a "latest wins" approach which can be used while -// merging two or more snapshots together. -func FilterLatestPackages(r *PackageRefList) *PackageRefList { +// FilterLatestRefs takes in a reflist with potentially multiples of the same +// packages and returns a reflist containing only the latest of each package. +// This implements a "latest wins" approach which can be used while merging two +// or more snapshots together. +func FilterLatestRefs(r *PackageRefList) *PackageRefList { // A running tab of latest seen package refs. latestRefs := make(map[string]int) - i := 0 - for _ = range r.Refs { + for i := 0; i < len(r.Refs); i++ { partsL := bytes.Split(r.Refs[i], []byte(" ")) archL, nameL, verL := partsL[0][1:], partsL[1], partsL[2] pkgId := string(nameL) + "." + string(archL) @@ -310,7 +301,6 @@ func FilterLatestPackages(r *PackageRefList) *PackageRefList { // If the package hasn't been seen before, add it and advance. if _, ok := latestRefs[pkgId]; !ok { latestRefs[pkgId] = i - i++ continue } @@ -320,13 +310,16 @@ func FilterLatestPackages(r *PackageRefList) *PackageRefList { vres := CompareVersions(string(verL), string(verR)) // Remove the older or duplicate refs from the result - if vres <= 0 { - r.Refs = append(r.Refs[0:i], r.Refs[i+1:]...) - latestRefs[pkgId] = i + if vres > 0 { + old := latestRefs[pkgId] + r.Refs = append(r.Refs[0:old], r.Refs[old+1:]...) + latestRefs[pkgId] = i - 1 } else { - oi := latestRefs[pkgId] - r.Refs = append(r.Refs[0:oi], r.Refs[oi+1:]...) + r.Refs = append(r.Refs[0:i], r.Refs[i+1:]...) } + + // Compensate for the reduced set + i -= 1 } return r From 708fd800dfd5f4a7095e5b80d48862b4bb724a68 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Tue, 22 Apr 2014 23:04:45 -0700 Subject: [PATCH 14/16] snapshot: separate test for FilterLatestRefs --- deb/reflist_test.go | 81 +++++++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 32 deletions(-) diff --git a/deb/reflist_test.go b/deb/reflist_test.go index 7393fa9e..c6fc867d 100644 --- a/deb/reflist_test.go +++ b/deb/reflist_test.go @@ -14,6 +14,14 @@ type PackageRefListSuite struct { var _ = Suite(&PackageRefListSuite{}) +func toStrSlice(reflist *PackageRefList) (result []string) { + result = make([]string, reflist.Len()) + for i, r := range reflist.Refs { + result[i] = string(r) + } + return +} + func (s *PackageRefListSuite) SetUpTest(c *C) { s.list = NewPackageList() @@ -247,49 +255,58 @@ func (s *PackageRefListSuite) TestMerge(c *C) { listB.Add(packages[5]) listB.Add(packages[6]) - listC := NewPackageList() - listC.Add(packages[3]) - listC.Add(packages[3]) - listC.Add(packages[4]) - listC.Add(packages[4]) - listC.Add(packages[7]) - listC.Add(packages[7]) - reflistA := NewPackageRefListFromPackageList(listA) reflistB := NewPackageRefListFromPackageList(listB) - reflistC := NewPackageRefListFromPackageList(listC) - toStrSlice := func(reflist *PackageRefList) (result []string) { - result = make([]string, reflist.Len()) - for i, r := range reflist.Refs { - result[i] = string(r) - } - return - } - - mergeAB := reflistA.Merge(reflistB, true, false) - mergeBA := reflistB.Merge(reflistA, true, false) + mergeAB := reflistA.Merge(reflistB, true) + mergeBA := reflistB.Merge(reflistA, true) c.Check(toStrSlice(mergeAB), DeepEquals, []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 lib 1.0", "Psparc xyz 1.0"}) c.Check(toStrSlice(mergeBA), DeepEquals, []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"}) - mergeABall := reflistA.Merge(reflistB, false, false) - mergeBAall := reflistB.Merge(reflistA, false, false) + mergeABall := reflistA.Merge(reflistB, false) + mergeBAall := reflistB.Merge(reflistA, false) c.Check(mergeABall, DeepEquals, mergeBAall) c.Check(toStrSlice(mergeBAall), DeepEquals, []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"}) - - mergeABClatest := reflistA.Merge(reflistB, false, true) - mergeABClatest = mergeABClatest.Merge(reflistC, false, true) - - mergeCBAlatest := reflistC.Merge(reflistB, false, true) - mergeCBAlatest = mergeCBAlatest.Merge(reflistA, false, true) - - c.Check(toStrSlice(mergeABClatest), DeepEquals, - []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"}) - c.Check(toStrSlice(mergeCBAlatest), DeepEquals, - []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"}) +} + +func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) { + packages := []*Package{ + &Package{Name: "lib", Version: "1.0", Architecture: "i386"}, + &Package{Name: "lib", Version: "1.1", Architecture: "i386"}, + &Package{Name: "lib", Version: "1.2", Architecture: "i386"}, + &Package{Name: "dpkg", Version: "1.2", Architecture: "i386"}, + &Package{Name: "dpkg", Version: "1.3", Architecture: "i386"}, + &Package{Name: "dpkg", Version: "1.4", Architecture: "i386"}, + &Package{Name: "dpkg", Version: "1.5", Architecture: "i386"}, + &Package{Name: "dpkg", Version: "1.6", Architecture: "i386"}, + } + + rl := NewPackageList() + rl.Add(packages[0]) + rl.Add(packages[1]) + rl.Add(packages[2]) + rl.Add(packages[3]) + rl.Add(packages[7]) + rl.Add(packages[0]) + rl.Add(packages[2]) + rl.Add(packages[4]) + rl.Add(packages[5]) + rl.Add(packages[6]) + rl.Add(packages[3]) + rl.Add(packages[3]) + rl.Add(packages[4]) + rl.Add(packages[4]) + rl.Add(packages[7]) + rl.Add(packages[7]) + + prl := NewPackageRefListFromPackageList(rl) + merged := FilterLatestRefs(prl) + + c.Check(toStrSlice(merged), DeepEquals, + []string{"Pi386 dpkg 1.6", "Pi386 lib 1.2"}) } From 6c3b2f686ec09857b6a09486e08806757b71a0ab Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Tue, 22 Apr 2014 23:33:30 -0700 Subject: [PATCH 15/16] snapshot: FilterLatestRefs returns nothing as it deals with a pointer. --- cmd/snapshot_merge.go | 2 +- deb/reflist.go | 10 +++++----- deb/reflist_test.go | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/snapshot_merge.go b/cmd/snapshot_merge.go index b7e85656..96ab10ac 100644 --- a/cmd/snapshot_merge.go +++ b/cmd/snapshot_merge.go @@ -38,7 +38,7 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error { } if latest { - result = deb.FilterLatestRefs(result) + deb.FilterLatestRefs(result) } sourceDescription := make([]string, len(sources)) diff --git a/deb/reflist.go b/deb/reflist.go index a6eb85cf..27636a5c 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -286,10 +286,10 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result } // FilterLatestRefs takes in a reflist with potentially multiples of the same -// packages and returns a reflist containing only the latest of each package. -// This implements a "latest wins" approach which can be used while merging two -// or more snapshots together. -func FilterLatestRefs(r *PackageRefList) *PackageRefList { +// packages and reduces it to only the latest of each package. The operations +// are done in-place. This implements a "latest wins" approach which can be used +// while merging two or more snapshots together. +func FilterLatestRefs(r *PackageRefList) { // A running tab of latest seen package refs. latestRefs := make(map[string]int) @@ -322,5 +322,5 @@ func FilterLatestRefs(r *PackageRefList) *PackageRefList { i -= 1 } - return r + return } diff --git a/deb/reflist_test.go b/deb/reflist_test.go index c6fc867d..12ef2711 100644 --- a/deb/reflist_test.go +++ b/deb/reflist_test.go @@ -304,9 +304,9 @@ func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) { rl.Add(packages[7]) rl.Add(packages[7]) - prl := NewPackageRefListFromPackageList(rl) - merged := FilterLatestRefs(prl) + result := NewPackageRefListFromPackageList(rl) + FilterLatestRefs(result) - c.Check(toStrSlice(merged), DeepEquals, + c.Check(toStrSlice(result), DeepEquals, []string{"Pi386 dpkg 1.6", "Pi386 lib 1.2"}) } From 3800f2c9577a64fc96b961e257cc5369c9a38019 Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Wed, 23 Apr 2014 00:25:44 -0700 Subject: [PATCH 16/16] Added CLI test for snapshot merge with latest flag --- system/t05_snapshot/MergeSnapshot6Test_gold | 3 +++ system/t05_snapshot/merge.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 system/t05_snapshot/MergeSnapshot6Test_gold diff --git a/system/t05_snapshot/MergeSnapshot6Test_gold b/system/t05_snapshot/MergeSnapshot6Test_gold new file mode 100644 index 00000000..eb356509 --- /dev/null +++ b/system/t05_snapshot/MergeSnapshot6Test_gold @@ -0,0 +1,3 @@ + +Snapshot snap4 successfully created. +You can run 'aptly publish snapshot snap4' to publish snapshot as Debian repository. diff --git a/system/t05_snapshot/merge.py b/system/t05_snapshot/merge.py index 3d25e788..842e7e63 100644 --- a/system/t05_snapshot/merge.py +++ b/system/t05_snapshot/merge.py @@ -78,3 +78,17 @@ class MergeSnapshot5Test(BaseTest): ] runCmd = "aptly snapshot merge snap1 snap1" expectedCode = 1 + + +class MergeSnapshot6Test(BaseTest): + """ + merge snapshots: use latest versions only + """ + fixtureDB = True + fixtureCmds = [ + "aptly snapshot create snap1 from mirror wheezy-main", + "aptly snapshot create snap2 from mirror wheezy-non-free", + "aptly snapshot create snap3 from mirror wheezy-backports", + ] + runCmd = "aptly snapshot merge -latest snap4 snap1 snap2 snap3" + expectedCode = 0