mirror update: add -max-versions to keep the N most recent versions

api/mirror.go

@@ -520,8 +520,10 @@ type mirrorUpdateParams struct {
 	ForceUpdate bool `            json:"ForceUpdate"`
 	// Set "true" to skip downloading already downloaded packages
 	SkipExistingPackages bool `   json:"SkipExistingPackages"`
-	// Set "true" to download only the latest version per package/architecture
+	// Set "true" to download only the latest version per package/architecture (same as `MaxVersions`: 1)
 	LatestOnly bool `             json:"LatestOnly"`
+	// Download only the N most recent versions per package/architecture, 0 means all versions
+	MaxVersions int `             json:"MaxVersions"            example:"5"`
 }
 
 // @Summary Update Mirror
@@ -573,6 +575,14 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 	}
 
+	if b.MaxVersions < 0 {
+		AbortWithJSONError(c, 400, fmt.Errorf("MaxVersions should be a positive number"))
+		return
+	}
+	if b.LatestOnly && b.MaxVersions == 0 {
+		b.MaxVersions = 1
+	}
+
 	verifier, err := getVerifier(b.Keyrings)
 	if err != nil {
 		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
@@ -640,8 +650,15 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}
 
+		if b.MaxVersions > 0 {
+			_, _, err := remote.FilterLatest(b.MaxVersions)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
 		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), taskCollectionFactory.PackageCollection(),
-			taskCollectionFactory.ChecksumCollection(nil), b.SkipExistingPackages, b.LatestOnly)
+			taskCollectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}

cmd/mirror_update.go

@@ -23,6 +23,14 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 
 	name := args[0]
 
+	maxVersions := context.Flags().Lookup("max-versions").Value.Get().(int)
+	if maxVersions < 0 {
+		return fmt.Errorf("unable to update: -max-versions should be a positive number")
+	}
+	if maxVersions == 0 && context.Flags().Lookup("latest").Value.Get().(bool) {
+		maxVersions = 1
+	}
+
 	collectionFactory := context.NewCollectionFactory()
 	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
@@ -90,17 +98,25 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen)
 	}
 
+	if maxVersions > 0 {
+		var oldLen, newLen int
+		oldLen, newLen, err = repo.FilterLatest(maxVersions)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+		context.Progress().Printf("Keeping %d latest version(s) of each package: %d -> %d.\n", maxVersions, oldLen, newLen)
+	}
+
 	var (
 		downloadSize int64
 		queue        []deb.PackageDownloadTask
 	)
 
 	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
-	latestOnly := context.Flags().Lookup("latest").Value.Get().(bool)
 
 	context.Progress().Printf("Building download queue...\n")
 	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
-		collectionFactory.ChecksumCollection(nil), skipExistingPackages, latestOnly)
+		collectionFactory.ChecksumCollection(nil), skipExistingPackages)
 
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -302,7 +318,8 @@ Example:
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
-	cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture)")
+	cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture), same as -max-versions=1")
+	cmd.Flag.Int("max-versions", 0, "download only the N most recent versions of each package (per architecture), 0 means all versions")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
 	cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
 	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")

deb/list.go

@@ -173,33 +173,21 @@ func (l *PackageList) ForEach(handler func(*Package) error) error {
 }
 
 // FilterLatest creates a copy of the package list containing only the
-// latest version for each package name/architecture pair.
-func (l *PackageList) FilterLatest() (*PackageList, error) {
+// count latest versions for each package name/architecture pair.
+func (l *PackageList) FilterLatest(count int) (*PackageList, error) {
 	if l == nil {
 		return nil, fmt.Errorf("package list is nil")
 	}
 
-	filtered := make(map[string]*Package, l.Len())
+	if count < 1 {
+		return nil, fmt.Errorf("number of latest versions should be at least 1, got %d", count)
+	}
+
+	groups := make(map[string][]*Package, l.Len())
 
 	err := l.ForEach(func(p *Package) error {
 		key := p.Architecture + "|" + p.Name
-
-		existing, found := filtered[key]
-		if !found {
-			filtered[key] = p
-			return nil
-		}
-
-		cmp, compareErr := CompareVersions(p.Version, existing.Version)
-		switch compareErr {
-		case nil:
-			if cmp > 0 {
-				filtered[key] = p
-			}
-		case RightInvalid:
-			filtered[key] = p
-		case LeftInvalid, BothInvalid:
-		}
+		groups[key] = append(groups[key], p)
 
 		return nil
 	})
@@ -207,11 +195,26 @@ func (l *PackageList) FilterLatest() (*PackageList, error) {
 		return nil, err
 	}
 
-	result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(filtered))
+	result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(groups)*count)
 
-	for _, pkg := range filtered {
-		if err = result.Add(pkg); err != nil {
-			return nil, err
+	for _, pkgs := range groups {
+		// sort from latest to oldest
+		sort.Slice(pkgs, func(i, j int) bool {
+			return CompareVersions(pkgs[i].Version, pkgs[j].Version) > 0
+		})
+
+		versions := 0
+		for i, p := range pkgs {
+			if i == 0 || CompareVersions(p.Version, pkgs[i-1].Version) != 0 {
+				versions++
+				if versions > count {
+					break
+				}
+			}
+
+			if err = result.Add(p); err != nil {
+				return nil, err
+			}
 		}
 	}
 

deb/remote.go

@@ -698,18 +698,26 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu
 	return
 }
 
-// BuildDownloadQueue builds queue, discards current PackageList
-func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages, latestOnly bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
+// FilterLatest keeps only count latest versions of each package (per architecture)
+func (repo *RemoteRepo) FilterLatest(count int) (oldLen, newLen int, err error) {
 	if repo.packageList == nil {
 		err = fmt.Errorf("package list is empty, please (re)download package indexes")
 		return
 	}
 
-	if latestOnly {
-		repo.packageList, err = repo.packageList.FilterLatest()
-		if err != nil {
-			return
-		}
+	oldLen = repo.packageList.Len()
+	repo.packageList, err = repo.packageList.FilterLatest(count)
+	if repo.packageList != nil {
+		newLen = repo.packageList.Len()
+	}
+	return
+}
+
+// BuildDownloadQueue builds queue, discards current PackageList
+func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
+	if repo.packageList == nil {
+		err = fmt.Errorf("package list is empty, please (re)download package indexes")
+		return
 	}
 
 	queue = make([]PackageDownloadTask, 0, repo.packageList.Len())