Merge pull request #1650 from Rascalov/mirror-update-max-versions

Add `-max-versions` to get the N most recent versions of a package.

api/mirror.go

@@ -520,8 +520,10 @@ type mirrorUpdateParams struct {
 	ForceUpdate bool `            json:"ForceUpdate"`
 	// Set "true" to skip downloading already downloaded packages
 	SkipExistingPackages bool `   json:"SkipExistingPackages"`
-	// Set "true" to download only the latest version per package/architecture
+	// Set "true" to download only the latest version per package/architecture (same as `MaxVersions`: 1)
 	LatestOnly bool `             json:"LatestOnly"`
+	// Download only the N most recent versions per package/architecture, 0 means all versions
+	MaxVersions int `             json:"MaxVersions"            example:"5"`
 }
 
 // @Summary Update Mirror
@@ -573,6 +575,14 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 	}
 
+	if b.MaxVersions < 0 {
+		AbortWithJSONError(c, 400, fmt.Errorf("MaxVersions should be a positive number"))
+		return
+	}
+	if b.LatestOnly && b.MaxVersions == 0 {
+		b.MaxVersions = 1
+	}
+
 	verifier, err := getVerifier(b.Keyrings)
 	if err != nil {
 		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
@@ -640,8 +650,15 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}
 
+		if b.MaxVersions > 0 {
+			_, _, err := remote.FilterLatest(b.MaxVersions)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
 		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), taskCollectionFactory.PackageCollection(),
-			taskCollectionFactory.ChecksumCollection(nil), b.SkipExistingPackages, b.LatestOnly)
+			taskCollectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}

cmd/mirror_update.go

@@ -23,6 +23,14 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 
 	name := args[0]
 
+	maxVersions := context.Flags().Lookup("max-versions").Value.Get().(int)
+	if maxVersions < 0 {
+		return fmt.Errorf("unable to update: -max-versions should be a positive number")
+	}
+	if maxVersions == 0 && context.Flags().Lookup("latest").Value.Get().(bool) {
+		maxVersions = 1
+	}
+
 	collectionFactory := context.NewCollectionFactory()
 	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
@@ -90,17 +98,29 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen)
 	}
 
+	if maxVersions > 0 {
+		var oldLen, newLen int
+		oldLen, newLen, err = repo.FilterLatest(maxVersions)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+		versionWord := "versions"
+		if maxVersions == 1 {
+			versionWord = "version"
+		}
+		context.Progress().Printf("Keeping at most %d %s of each package: %d -> %d.\n", maxVersions, versionWord, oldLen, newLen)
+	}
+
 	var (
 		downloadSize int64
 		queue        []deb.PackageDownloadTask
 	)
 
 	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
-	latestOnly := context.Flags().Lookup("latest").Value.Get().(bool)
 
 	context.Progress().Printf("Building download queue...\n")
 	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
-		collectionFactory.ChecksumCollection(nil), skipExistingPackages, latestOnly)
+		collectionFactory.ChecksumCollection(nil), skipExistingPackages)
 
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -302,7 +322,8 @@ Example:
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
-	cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture)")
+	cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture), same as -max-versions=1")
+	cmd.Flag.Int("max-versions", 0, "download only the N most recent versions of each package (per architecture), 0 means all versions")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
 	cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
 	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")

completion.d/_aptly

@@ -212,7 +212,8 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             $keyring \
                             "-max-tries=[max download tries till process fails with download error]:number: " \
                             "-skip-existing-packages=[do not check file existence for packages listed in the internal database of the mirror]:$bool" \
-                            "-latest=[download only latest version of each package (per architecture)]:$bool" \
+                            "-latest=[download only latest version of each package (per architecture), same as -max-versions=1]:$bool" \
+                            "-max-versions=[download only the N most recent versions of each package (per architecture), 0 means all versions]:number: " \
                             "(-)2:mirror name:$mirrors"
                         ;;
                     rename)

completion.d/aptly

@@ -263,7 +263,7 @@ _aptly()
           "update")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages -latest" -- ${cur}))
+                COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages -latest -max-versions=" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
               fi

deb/list.go

@@ -173,33 +173,21 @@ func (l *PackageList) ForEach(handler func(*Package) error) error {
 }
 
 // FilterLatest creates a copy of the package list containing only the
-// latest version for each package name/architecture pair.
-func (l *PackageList) FilterLatest() (*PackageList, error) {
+// count latest versions for each package name/architecture pair.
+func (l *PackageList) FilterLatest(count int) (*PackageList, error) {
 	if l == nil {
 		return nil, fmt.Errorf("package list is nil")
 	}
 
-	filtered := make(map[string]*Package, l.Len())
+	if count < 1 {
+		return nil, fmt.Errorf("number of latest versions should be at least 1, got %d", count)
+	}
+
+	groups := make(map[string][]*Package, l.Len())
 
 	err := l.ForEach(func(p *Package) error {
 		key := p.Architecture + "|" + p.Name
-
-		existing, found := filtered[key]
-		if !found {
-			filtered[key] = p
-			return nil
-		}
-
-		cmp, compareErr := CompareVersions(p.Version, existing.Version)
-		switch compareErr {
-		case nil:
-			if cmp > 0 {
-				filtered[key] = p
-			}
-		case RightInvalid:
-			filtered[key] = p
-		case LeftInvalid, BothInvalid:
-		}
+		groups[key] = append(groups[key], p)
 
 		return nil
 	})
@@ -207,11 +195,49 @@ func (l *PackageList) FilterLatest() (*PackageList, error) {
 		return nil, err
 	}
 
-	result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(filtered))
+	result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(groups)*count)
 
-	for _, pkg := range filtered {
-		if err = result.Add(pkg); err != nil {
-			return nil, err
+	for _, pkgs := range groups {
+		// sort from latest to oldest
+		sort.SliceStable(pkgs, func(i, j int) bool {
+			cmp, cmpErr := CompareVersions(pkgs[i].Version, pkgs[j].Version)
+			switch cmpErr {
+			case LeftInvalid, BothInvalid:
+				return false
+			case RightInvalid:
+				return true
+			}
+			return cmp > 0
+		})
+
+		versions := 0
+		for i, p := range pkgs {
+			if i == 0 {
+				// first element always included
+			} else {
+				cmp, cmpErr := CompareVersions(p.Version, pkgs[i-1].Version)
+				switch cmpErr {
+				case LeftInvalid, BothInvalid:
+					// invalid version is different from previous
+					versions++
+				case RightInvalid:
+					// previous was invalid, current is valid — different
+					versions++
+				default:
+					if cmp == 0 {
+						// same version, skip
+					} else {
+						versions++
+					}
+				}
+				if versions >= count {
+					break
+				}
+			}
+
+			if err = result.Add(p); err != nil {
+				return nil, err
+			}
 		}
 	}
 

deb/list_test.go

@@ -524,13 +524,55 @@ func (s *PackageListSuite) TestFilterLatest(c *C) {
 	sharedPkg := NewPackageFromControlFile(shared)
 	_ = list.Add(sharedPkg)
 
-	filtered, err := list.FilterLatest()
+	filtered, err := list.FilterLatest(1)
 	c.Assert(err, IsNil)
 	c.Assert(filtered.Len(), Equals, 2)
 	c.Check(filtered.Has(newerPkg), Equals, true)
 	c.Check(filtered.Has(sharedPkg), Equals, true)
 }
 
+func (s *PackageListSuite) TestFilterLatestMultiple(c *C) {
+	list := NewPackageList()
+
+	pkgs := make([]*Package, 0, 4)
+	for _, version := range []string{"2.0", "1.0", "10.0", "1:0.1"} {
+		stanza := packageStanza.Copy()
+		stanza["Version"] = version
+		pkg := NewPackageFromControlFile(stanza)
+		_ = list.Add(pkg)
+		pkgs = append(pkgs, pkg)
+	}
+
+	shared := packageStanza.Copy()
+	shared["Architecture"] = ArchitectureAll
+	shared["Version"] = "3.0"
+	shared["Package"] = "shared"
+	sharedPkg := NewPackageFromControlFile(shared)
+	_ = list.Add(sharedPkg)
+
+	filtered, err := list.FilterLatest(3)
+	c.Assert(err, IsNil)
+	c.Assert(filtered.Len(), Equals, 4)
+	c.Check(filtered.Has(pkgs[0]), Equals, true)
+	c.Check(filtered.Has(pkgs[1]), Equals, false)
+	c.Check(filtered.Has(pkgs[2]), Equals, true)
+	c.Check(filtered.Has(pkgs[3]), Equals, true)
+	c.Check(filtered.Has(sharedPkg), Equals, true)
+
+	// fewer versions available than requested: everything is kept
+	filtered, err = list.FilterLatest(5)
+	c.Assert(err, IsNil)
+	c.Check(filtered.Len(), Equals, 5)
+}
+
+func (s *PackageListSuite) TestFilterLatestInvalidCount(c *C) {
+	list := NewPackageList()
+
+	filtered, err := list.FilterLatest(0)
+	c.Assert(err, ErrorMatches, "number of latest versions should be at least 1, got 0")
+	c.Assert(filtered, IsNil)
+}
+
 func (s *PackageListSuite) TestFilterLatestInvalidVersions(c *C) {
 	list := NewPackageList()
 	valid := &Package{Name: "pkg", Version: "1.0", Architecture: "amd64"}
@@ -538,7 +580,7 @@ func (s *PackageListSuite) TestFilterLatestInvalidVersions(c *C) {
 	_ = list.Add(valid)
 	_ = list.Add(invalid)
 
-	filtered, err := list.FilterLatest()
+	filtered, err := list.FilterLatest(1)
 	c.Assert(err, IsNil)
 	c.Check(filtered.Len(), Equals, 1)
 	c.Check(filtered.Has(valid), Equals, true)
@@ -546,7 +588,7 @@ func (s *PackageListSuite) TestFilterLatestInvalidVersions(c *C) {
 	invalidOnly := NewPackageList()
 	_ = invalidOnly.Add(&Package{Name: "pkg", Version: "1.0-", Architecture: "amd64"})
 	_ = invalidOnly.Add(&Package{Name: "pkg", Version: "2.0-", Architecture: "amd64"})
-	filtered, err = invalidOnly.FilterLatest()
+	filtered, err = invalidOnly.FilterLatest(1)
 	c.Assert(err, IsNil)
 	c.Check(filtered.Len(), Equals, 1)
 }
@@ -570,7 +612,7 @@ func (s *PackageListSuite) TestFilterLatestPreservesDuplicatesFlag(c *C) {
 	another["Version"] = "7.41-1"
 	_ = list.Add(NewPackageFromControlFile(another))
 
-	filtered, err := list.FilterLatest()
+	filtered, err := list.FilterLatest(1)
 	c.Assert(err, IsNil)
 	c.Assert(filtered.duplicatesAllowed, Equals, true)
 }
@@ -578,7 +620,7 @@ func (s *PackageListSuite) TestFilterLatestPreservesDuplicatesFlag(c *C) {
 func (s *PackageListSuite) TestFilterLatestNil(c *C) {
 	var list *PackageList
 
-	filtered, err := list.FilterLatest()
+	filtered, err := list.FilterLatest(1)
 	c.Assert(err, ErrorMatches, "package list is nil")
 	c.Assert(filtered, IsNil)
 }

deb/remote.go

@@ -698,18 +698,26 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu
 	return
 }
 
-// BuildDownloadQueue builds queue, discards current PackageList
-func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages, latestOnly bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
+// FilterLatest keeps only count latest versions of each package (per architecture)
+func (repo *RemoteRepo) FilterLatest(count int) (oldLen, newLen int, err error) {
 	if repo.packageList == nil {
 		err = fmt.Errorf("package list is empty, please (re)download package indexes")
 		return
 	}
 
-	if latestOnly {
-		repo.packageList, err = repo.packageList.FilterLatest()
-		if err != nil {
-			return
-		}
+	oldLen = repo.packageList.Len()
+	repo.packageList, err = repo.packageList.FilterLatest(count)
+	if repo.packageList != nil {
+		newLen = repo.packageList.Len()
+	}
+	return
+}
+
+// BuildDownloadQueue builds queue, discards current PackageList
+func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
+	if repo.packageList == nil {
+		err = fmt.Errorf("package list is empty, please (re)download package indexes")
+		return
 	}
 
 	queue = make([]PackageDownloadTask, 0, repo.packageList.Len())

deb/remote_test.go

@@ -322,7 +322,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
 	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(3))
 	c.Check(queue, HasLen, 1)
@@ -349,7 +349,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false)
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
 	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(0))
 	c.Check(queue, HasLen, 0)
@@ -370,7 +370,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
 	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(3))
 	c.Check(queue, HasLen, 1)
@@ -397,7 +397,7 @@ func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
 	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(3)+int64(len(exampleInstallerManifestFile)))
 	c.Check(queue, HasLen, 2)
@@ -423,7 +423,7 @@ func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) {
 	c.Check(pkg.Name, Equals, "installer")
 }
 
-func (s *RemoteRepoSuite) TestBuildDownloadQueueLatestOnly(c *C) {
+func (s *RemoteRepoSuite) TestFilterLatest(c *C) {
 	s.repo.Architectures = []string{"i386"}
 
 	err := s.repo.Fetch(s.downloader, nil, true)
@@ -445,13 +445,29 @@ func (s *RemoteRepoSuite) TestBuildDownloadQueueLatestOnly(c *C) {
 	newest := NewPackageFromControlFile(stanza)
 	_ = s.repo.packageList.Add(newest)
 
-	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, true)
+	oldLen, newLen, err := s.repo.FilterLatest(1)
+	c.Assert(err, IsNil)
+	c.Check(oldLen, Equals, 2)
+	c.Check(newLen, Equals, 1)
+
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
 	c.Assert(err, IsNil)
 	c.Check(queue, HasLen, 1)
 	c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.4.0-1_i386.deb")
 	c.Check(size, Equals, int64(187518))
 }
 
+func (s *RemoteRepoSuite) TestFilterLatestErrors(c *C) {
+	_, _, err := s.repo.FilterLatest(1)
+	c.Assert(err, ErrorMatches, "package list is empty, please \\(re\\)download package indexes")
+
+	s.repo.packageList = NewPackageList()
+	oldLen, newLen, err := s.repo.FilterLatest(0)
+	c.Assert(err, ErrorMatches, "number of latest versions should be at least 1, got 0")
+	c.Check(oldLen, Equals, 0)
+	c.Check(newLen, Equals, 0)
+}
+
 func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.repo.Architectures = []string{"i386"}
 	s.repo.DownloadSources = true
@@ -470,7 +486,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
 	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(15))
 	c.Check(queue, HasLen, 4)
@@ -514,7 +530,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false)
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
 	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(0))
 	c.Check(queue, HasLen, 0)
@@ -539,7 +555,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
 	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(15))
 	c.Check(queue, HasLen, 4)
@@ -563,7 +579,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
 	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(3))
 	c.Check(queue, HasLen, 1)
@@ -591,7 +607,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false)
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
 	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(0))
 	c.Check(queue, HasLen, 0)
@@ -613,7 +629,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
 	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(3))
 	c.Check(queue, HasLen, 1)
@@ -644,7 +660,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
 	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(15))
 	c.Check(queue, HasLen, 4)
@@ -690,7 +706,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false)
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
 	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(0))
 	c.Check(queue, HasLen, 0)
@@ -716,7 +732,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
 	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(15))
 	c.Check(queue, HasLen, 4)

man/aptly.1

@@ -825,7 +825,11 @@ do not check file existence for packages listed in the internal database of the
 .
 .TP
 \-\fBlatest\fR
-download only latest version of each package (per architecture)
+download only latest version of each package (per architecture), same as \-max\-versions=1
+.
+.TP
+\-\fBmax\-versions\fR=0
+download only the N most recent versions of each package (per architecture), 0 means all versions
 .
 .SH "RENAMES MIRROR"
 \fBaptly\fR \fBmirror\fR \fBrename\fR \fIold\-name\fR \fInew\-name\fR

system/t04_mirror/UpdateMirror29Test_gold

@@ -0,0 +1,32 @@
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/Release
+Downloading & parsing package files...
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-i386/Packages.bz2
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-amd64/Packages.bz2
+Keeping at most 2 versions of each package: 52 -> 23.
+Building download queue...
+Download queue: 23 items (8.41 MiB)
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish_3.0.6-1~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.1%29/varnish-agent_2.2.1~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%283.0.0%29/varnish-agent_3.0.0~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-dbg_3.0.7-1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi1_3.0.6-1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi-dev_3.0.7-1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi1_3.0.7-1~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi1_3.0.6-1~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish_3.0.7-1~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.0%29/varnish-agent_2.2.0~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi-dev_3.0.7-1~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish_3.0.7-1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish_3.0.6-1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi-dev_3.0.6-1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%283.0.1%29/varnish-agent_3.0.1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish-dbg_3.0.6-1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi-dev_3.0.6-1~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-dbg_3.0.7-1~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-doc_3.0.7-1~wheezy_all.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish-doc_3.0.6-1~wheezy_all.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi1_3.0.7-1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%281.16.0%29/varnish-agent_1.16.0~wheezy_all.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish-dbg_3.0.6-1~wheezy_i386.deb
+
+Mirror `varnish` has been updated successfully.

system/t04_mirror/UpdateMirror29Test_mirror_show

@@ -0,0 +1,45 @@
+Name: varnish
+Archive Root URL: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/
+Distribution: wheezy
+Components: main
+Architectures: i386, amd64
+Download Sources: no
+Download .udebs: no
+Download AppStream: no
+Number of packages: 23
+
+Information from release file:
+Acquire-By-Hash: yes
+Architectures: amd64 armel armhf i386 ia64 mips mipsel powerpc ppc64el s390 s390x sparc alpha avr32 hppa m32 m68k sh arm64
+Codename: wheezy
+Components: main
+Date: Tue, 22 Jan 2019 00:28:39 +0000
+Description: 
+Label: varnish30
+Origin: packagecloud.io/varnishcache/varnish30
+Suite: wheezy
+Version: 1
+Packages:
+  libvarnishapi-dev_3.0.7-1~wheezy_amd64
+  libvarnishapi-dev_3.0.7-1~wheezy_i386
+  libvarnishapi-dev_3.0.6-1~wheezy_amd64
+  libvarnishapi-dev_3.0.6-1~wheezy_i386
+  libvarnishapi1_3.0.7-1~wheezy_amd64
+  libvarnishapi1_3.0.7-1~wheezy_i386
+  libvarnishapi1_3.0.6-1~wheezy_amd64
+  libvarnishapi1_3.0.6-1~wheezy_i386
+  varnish_3.0.7-1~wheezy_amd64
+  varnish_3.0.7-1~wheezy_i386
+  varnish_3.0.6-1~wheezy_amd64
+  varnish_3.0.6-1~wheezy_i386
+  varnish-agent_3.0.1~wheezy_amd64
+  varnish-agent_3.0.0~wheezy_amd64
+  varnish-agent_2.2.1~wheezy_i386
+  varnish-agent_2.2.0~wheezy_i386
+  varnish-agent_1.16.0~wheezy_all
+  varnish-dbg_3.0.7-1~wheezy_amd64
+  varnish-dbg_3.0.7-1~wheezy_i386
+  varnish-dbg_3.0.6-1~wheezy_amd64
+  varnish-dbg_3.0.6-1~wheezy_i386
+  varnish-doc_3.0.7-1~wheezy_all
+  varnish-doc_3.0.6-1~wheezy_all

system/t04_mirror/UpdateMirror30Test_gold

@@ -0,0 +1,21 @@
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/Release
+Downloading & parsing package files...
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-i386/Packages.bz2
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-amd64/Packages.bz2
+Keeping at most 1 version of each package: 52 -> 12.
+Building download queue...
+Download queue: 12 items (4.19 MiB)
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-dbg_3.0.7-1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi1_3.0.7-1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%281.16.0%29/varnish-agent_1.16.0~wheezy_all.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%283.0.1%29/varnish-agent_3.0.1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish_3.0.7-1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-dbg_3.0.7-1~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish_3.0.7-1~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi-dev_3.0.7-1~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi1_3.0.7-1~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi-dev_3.0.7-1~wheezy_amd64.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.1%29/varnish-agent_2.2.1~wheezy_i386.deb
+Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-doc_3.0.7-1~wheezy_all.deb
+
+Mirror `varnish` has been updated successfully.

system/t04_mirror/UpdateMirror30Test_mirror_show

@@ -0,0 +1,34 @@
+Name: varnish
+Archive Root URL: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/
+Distribution: wheezy
+Components: main
+Architectures: i386, amd64
+Download Sources: no
+Download .udebs: no
+Download AppStream: no
+Number of packages: 12
+
+Information from release file:
+Acquire-By-Hash: yes
+Architectures: amd64 armel armhf i386 ia64 mips mipsel powerpc ppc64el s390 s390x sparc alpha avr32 hppa m32 m68k sh arm64
+Codename: wheezy
+Components: main
+Date: Tue, 22 Jan 2019 00:28:39 +0000
+Description: 
+Label: varnish30
+Origin: packagecloud.io/varnishcache/varnish30
+Suite: wheezy
+Version: 1
+Packages:
+  libvarnishapi-dev_3.0.7-1~wheezy_amd64
+  libvarnishapi-dev_3.0.7-1~wheezy_i386
+  libvarnishapi1_3.0.7-1~wheezy_amd64
+  libvarnishapi1_3.0.7-1~wheezy_i386
+  varnish_3.0.7-1~wheezy_amd64
+  varnish_3.0.7-1~wheezy_i386
+  varnish-agent_3.0.1~wheezy_amd64
+  varnish-agent_2.2.1~wheezy_i386
+  varnish-agent_1.16.0~wheezy_all
+  varnish-dbg_3.0.7-1~wheezy_amd64
+  varnish-dbg_3.0.7-1~wheezy_i386
+  varnish-doc_3.0.7-1~wheezy_all

system/t04_mirror/UpdateMirror31Test_gold

@@ -0,0 +1 @@
+ERROR: unable to update: -max-versions should be a positive number

system/t04_mirror/update.py

@@ -502,3 +502,45 @@ class UpdateMirror28Test(BaseTest):
         self.check_output()
         self.check_cmd_output("aptly mirror show appstream-test", "mirror_show",
                               match_prepare=lambda s: re.sub(r"Last update: [0-9:+A-Za-z -]+\n", "", s))
+
+
+class UpdateMirror29Test(BaseTest):
+    """
+    update mirrors: keep only the 2 most recent versions of each package
+    """
+    sortOutput = True
+    fixtureCmds = [
+        "aptly -architectures=i386,amd64 mirror create --ignore-signatures varnish http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/ wheezy main",
+    ]
+    runCmd = "aptly mirror update --ignore-signatures -max-versions=2 varnish"
+    outputMatchPrepare = filterOutRedirects
+
+    def check(self):
+        self.check_output()
+        self.check_cmd_output("aptly mirror show -with-packages varnish", "mirror_show",
+                              match_prepare=lambda s: re.sub(r"Last update: [0-9:+A-Za-z -]+\n", "", s))
+
+
+class UpdateMirror30Test(BaseTest):
+    """
+    update mirrors: -latest keeps only the most recent version of each package
+    """
+    sortOutput = True
+    fixtureCmds = [
+        "aptly -architectures=i386,amd64 mirror create --ignore-signatures varnish http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/ wheezy main",
+    ]
+    runCmd = "aptly mirror update --ignore-signatures -latest varnish"
+    outputMatchPrepare = filterOutRedirects
+
+    def check(self):
+        self.check_output()
+        self.check_cmd_output("aptly mirror show -with-packages varnish", "mirror_show",
+                              match_prepare=lambda s: re.sub(r"Last update: [0-9:+A-Za-z -]+\n", "", s))
+
+
+class UpdateMirror31Test(BaseTest):
+    """
+    update mirrors: negative -max-versions
+    """
+    runCmd = "aptly mirror update -max-versions=-1 varnish"
+    expectedCode = 1

system/t12_api/mirrors.py

@@ -372,3 +372,47 @@ class MirrorsAPITestEditFlatMirrorUdebs(APITest):
         resp = self.post("/api/mirrors/" + mirror_name, json=edit_params)
         self.check_equal(resp.status_code, 400)
         self.check_in("flat mirrors don't support udebs", resp.json()['error'])
+
+
+class MirrorsAPITestUpdateMaxVersions(APITest):
+    """
+    PUT /api/mirrors/:name with MaxVersions and LatestOnly
+    """
+    def check(self):
+        mirror_name = self.random_name()
+        mirror_desc = {'Name': mirror_name,
+                       'ArchiveURL': 'http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/',
+                       'Distribution': 'wheezy',
+                       'Architectures': ["amd64"],
+                       'Components': ['main'],
+                       'IgnoreSignatures': True}
+
+        resp = self.post("/api/mirrors", json=mirror_desc)
+        self.check_equal(resp.status_code, 201)
+
+        def versions_per_package():
+            resp = self.get("/api/mirrors/" + mirror_name + "/packages")
+            self.check_equal(resp.status_code, 200)
+            versions = {}
+            for key in resp.json():
+                arch, name, version = key.split(" ")[:3]
+                versions.setdefault((arch, name), []).append(version)
+            return versions
+
+        resp = self.put("/api/mirrors/" + mirror_name, json={'IgnoreSignatures': True, 'MaxVersions': -1})
+        self.check_equal(resp.status_code, 400)
+        self.check_in("MaxVersions should be a positive number", resp.json()['error'])
+
+        resp = self.put_task("/api/mirrors/" + mirror_name, json={'IgnoreSignatures': True, 'MaxVersions': 2})
+        self.check_task(resp)
+
+        versions = versions_per_package()
+        self.check_equal(sorted(versions[("Pamd64", "varnish")]), ["3.0.6-1~wheezy", "3.0.7-1~wheezy"])
+        self.check_equal(max(len(v) for v in versions.values()), 2)
+
+        resp = self.put_task("/api/mirrors/" + mirror_name, json={'IgnoreSignatures': True, 'LatestOnly': True})
+        self.check_task(resp)
+
+        versions = versions_per_package()
+        self.check_equal(versions[("Pamd64", "varnish")], ["3.0.7-1~wheezy"])
+        self.check_equal(max(len(v) for v in versions.values()), 1)