diff --git a/api/mirror.go b/api/mirror.go index 80aa8e0a..e5a00aac 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -520,8 +520,10 @@ type mirrorUpdateParams struct { ForceUpdate bool ` json:"ForceUpdate"` // Set "true" to skip downloading already downloaded packages SkipExistingPackages bool ` json:"SkipExistingPackages"` - // Set "true" to download only the latest version per package/architecture + // Set "true" to download only the latest version per package/architecture (same as `MaxVersions`: 1) LatestOnly bool ` json:"LatestOnly"` + // Download only the N most recent versions per package/architecture, 0 means all versions + MaxVersions int ` json:"MaxVersions" example:"5"` } // @Summary Update Mirror @@ -573,6 +575,14 @@ func apiMirrorsUpdate(c *gin.Context) { } } + if b.MaxVersions < 0 { + AbortWithJSONError(c, 400, fmt.Errorf("MaxVersions should be a positive number")) + return + } + if b.LatestOnly && b.MaxVersions == 0 { + b.MaxVersions = 1 + } + verifier, err := getVerifier(b.Keyrings) if err != nil { AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err)) @@ -640,8 +650,15 @@ func apiMirrorsUpdate(c *gin.Context) { } } + if b.MaxVersions > 0 { + _, _, err := remote.FilterLatest(b.MaxVersions) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + } + queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), taskCollectionFactory.PackageCollection(), - taskCollectionFactory.ChecksumCollection(nil), b.SkipExistingPackages, b.LatestOnly) + taskCollectionFactory.ChecksumCollection(nil), b.SkipExistingPackages) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } diff --git a/cmd/mirror_update.go b/cmd/mirror_update.go index d67fce3e..f776fa59 100644 --- a/cmd/mirror_update.go +++ b/cmd/mirror_update.go @@ -23,6 +23,14 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { name := args[0] + maxVersions := context.Flags().Lookup("max-versions").Value.Get().(int) + if maxVersions < 0 { + return fmt.Errorf("unable to update: -max-versions should be a positive number") + } + if maxVersions == 0 && context.Flags().Lookup("latest").Value.Get().(bool) { + maxVersions = 1 + } + collectionFactory := context.NewCollectionFactory() repo, err := collectionFactory.RemoteRepoCollection().ByName(name) if err != nil { @@ -90,17 +98,25 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen) } + if maxVersions > 0 { + var oldLen, newLen int + oldLen, newLen, err = repo.FilterLatest(maxVersions) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + context.Progress().Printf("Keeping %d latest version(s) of each package: %d -> %d.\n", maxVersions, oldLen, newLen) + } + var ( downloadSize int64 queue []deb.PackageDownloadTask ) skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool) - latestOnly := context.Flags().Lookup("latest").Value.Get().(bool) context.Progress().Printf("Building download queue...\n") queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(), - collectionFactory.ChecksumCollection(nil), skipExistingPackages, latestOnly) + collectionFactory.ChecksumCollection(nil), skipExistingPackages) if err != nil { return fmt.Errorf("unable to update: %s", err) @@ -302,7 +318,8 @@ Example: cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata") cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror") - cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture)") + cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture), same as -max-versions=1") + cmd.Flag.Int("max-versions", 0, "download only the N most recent versions of each package (per architecture), 0 means all versions") cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)") cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)") cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error") diff --git a/deb/list.go b/deb/list.go index 7395734b..5806f950 100644 --- a/deb/list.go +++ b/deb/list.go @@ -173,33 +173,21 @@ func (l *PackageList) ForEach(handler func(*Package) error) error { } // FilterLatest creates a copy of the package list containing only the -// latest version for each package name/architecture pair. -func (l *PackageList) FilterLatest() (*PackageList, error) { +// count latest versions for each package name/architecture pair. +func (l *PackageList) FilterLatest(count int) (*PackageList, error) { if l == nil { return nil, fmt.Errorf("package list is nil") } - filtered := make(map[string]*Package, l.Len()) + if count < 1 { + return nil, fmt.Errorf("number of latest versions should be at least 1, got %d", count) + } + + groups := make(map[string][]*Package, l.Len()) err := l.ForEach(func(p *Package) error { key := p.Architecture + "|" + p.Name - - existing, found := filtered[key] - if !found { - filtered[key] = p - return nil - } - - cmp, compareErr := CompareVersions(p.Version, existing.Version) - switch compareErr { - case nil: - if cmp > 0 { - filtered[key] = p - } - case RightInvalid: - filtered[key] = p - case LeftInvalid, BothInvalid: - } + groups[key] = append(groups[key], p) return nil }) @@ -207,11 +195,26 @@ func (l *PackageList) FilterLatest() (*PackageList, error) { return nil, err } - result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(filtered)) + result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(groups)*count) - for _, pkg := range filtered { - if err = result.Add(pkg); err != nil { - return nil, err + for _, pkgs := range groups { + // sort from latest to oldest + sort.Slice(pkgs, func(i, j int) bool { + return CompareVersions(pkgs[i].Version, pkgs[j].Version) > 0 + }) + + versions := 0 + for i, p := range pkgs { + if i == 0 || CompareVersions(p.Version, pkgs[i-1].Version) != 0 { + versions++ + if versions > count { + break + } + } + + if err = result.Add(p); err != nil { + return nil, err + } } } diff --git a/deb/remote.go b/deb/remote.go index a1a79d68..37966124 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -698,18 +698,26 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu return } -// BuildDownloadQueue builds queue, discards current PackageList -func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages, latestOnly bool) (queue []PackageDownloadTask, downloadSize int64, err error) { +// FilterLatest keeps only count latest versions of each package (per architecture) +func (repo *RemoteRepo) FilterLatest(count int) (oldLen, newLen int, err error) { if repo.packageList == nil { err = fmt.Errorf("package list is empty, please (re)download package indexes") return } - if latestOnly { - repo.packageList, err = repo.packageList.FilterLatest() - if err != nil { - return - } + oldLen = repo.packageList.Len() + repo.packageList, err = repo.packageList.FilterLatest(count) + if repo.packageList != nil { + newLen = repo.packageList.Len() + } + return +} + +// BuildDownloadQueue builds queue, discards current PackageList +func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) { + if repo.packageList == nil { + err = fmt.Errorf("package list is empty, please (re)download package indexes") + return } queue = make([]PackageDownloadTask, 0, repo.packageList.Len())