vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-01-11 03:11:50 +00:00
Code Issues Projects Releases Wiki Activity

Harden latest-only filtering

Browse Source
This commit is contained in:
Juan Calderon-Perez
2025-12-11 07:20:37 -05:00
parent ba65daf6cb
commit 0021cf876b
9 changed files with 149 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
api/mirror.go
View File

@@ -343,6 +343,8 @@ type mirrorUpdateParams struct {
ForceUpdate bool ` json:"ForceUpdate"`
// Set "true" to skip downloading already downloaded packages
SkipExistingPackages bool ` json:"SkipExistingPackages"`
// Set "true" to download only the latest version per package/architecture
LatestOnly bool ` json:"LatestOnly"`
}
// @Summary Update Mirror
@@ -434,7 +436,7 @@ func apiMirrorsUpdate(c *gin.Context) {
}
queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages, b.LatestOnly)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}

4
cmd/mirror_update.go
View File

@@ -87,10 +87,11 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
)
skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
latestOnly := context.Flags().Lookup("latest").Value.Get().(bool)
context.Progress().Printf("Building download queue...\n")
queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
collectionFactory.ChecksumCollection(nil), skipExistingPackages)
collectionFactory.ChecksumCollection(nil), skipExistingPackages, latestOnly)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
@@ -292,6 +293,7 @@ Example:
cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture)")
cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")

1
completion.d/_aptly
View File

@@ -211,6 +211,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
$keyring \
"-max-tries=[max download tries till process fails with download error]:number: " \
"-skip-existing-packages=[do not check file existence for packages listed in the internal database of the mirror]:$bool" \
"-latest=[download only latest version of each package (per architecture)]:$bool" \
"(-)2:mirror name:$mirrors"
;;
rename)

2
completion.d/aptly
View File

@@ -263,7 +263,7 @@ _aptly()
"update")
if [[ $numargs -eq 0 ]]; then
if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur}))
COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages -latest" -- ${cur}))
else
COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
fi

33
deb/list.go
View File

@@ -172,6 +172,39 @@ func (l *PackageList) ForEach(handler func(*Package) error) error {
return err
}
// FilterLatest creates a copy of the package list containing only the
// latest version for each package name/architecture pair.
func (l *PackageList) FilterLatest() (*PackageList, error) {
if l == nil {
return nil, fmt.Errorf("package list is nil")
}
filtered := make(map[string]*Package, l.Len())
err := l.ForEach(func(p *Package) error {
key := p.Architecture + "|" + p.Name
if existing, found := filtered[key]; !found || CompareVersions(p.Version, existing.Version) > 0 {
filtered[key] = p
}
return nil
})
if err != nil {
return nil, err
}
result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(filtered))
for _, pkg := range filtered {
if err = result.Add(pkg); err != nil {
return nil, err
}
}
return result, nil
}
// ForEachIndexed calls handler for each package in list in indexed order
func (l *PackageList) ForEachIndexed(handler func(*Package) error) error {
if !l.indexed {

49
deb/list_test.go
View File

@@ -503,3 +503,52 @@ func (s *PackageListSuite) TestArchitectures(c *C) {
sort.Strings(archs)
c.Check(archs, DeepEquals, []string{"amd64", "arm", "i386", "s390"})
}
func (s *PackageListSuite) TestFilterLatest(c *C) {
list := NewPackageList()
older := packageStanza.Copy()
older["Version"] = "1.0"
olderPkg := NewPackageFromControlFile(older)
_ = list.Add(olderPkg)
newer := packageStanza.Copy()
newer["Version"] = "2.0"
newerPkg := NewPackageFromControlFile(newer)
_ = list.Add(newerPkg)
shared := packageStanza.Copy()
shared["Architecture"] = ArchitectureAll
shared["Version"] = "3.0"
shared["Package"] = "shared"
sharedPkg := NewPackageFromControlFile(shared)
_ = list.Add(sharedPkg)
filtered, err := list.FilterLatest()
c.Assert(err, IsNil)
c.Assert(filtered.Len(), Equals, 2)
c.Check(filtered.Has(newerPkg), Equals, true)
c.Check(filtered.Has(sharedPkg), Equals, true)
}
func (s *PackageListSuite) TestFilterLatestPreservesDuplicatesFlag(c *C) {
list := NewPackageListWithDuplicates(true, 2)
_ = list.Add(NewPackageFromControlFile(packageStanza.Copy()))
another := packageStanza.Copy()
another["Version"] = "7.41-1"
_ = list.Add(NewPackageFromControlFile(another))
filtered, err := list.FilterLatest()
c.Assert(err, IsNil)
c.Assert(filtered.duplicatesAllowed, Equals, true)
}
func (s *PackageListSuite) TestFilterLatestNil(c *C) {
var list *PackageList
filtered, err := list.FilterLatest()
c.Assert(err, ErrorMatches, "package list is nil")
c.Assert(filtered, IsNil)
}

14
deb/remote.go
View File

@@ -612,7 +612,19 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu
}
// BuildDownloadQueue builds queue, discards current PackageList
func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages, latestOnly bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
if repo.packageList == nil {
err = fmt.Errorf("package list is empty, please (re)download package indexes")
return
}
if latestOnly {
repo.packageList, err = repo.packageList.FilterLatest()
if err != nil {
return
}
}
queue = make([]PackageDownloadTask, 0, repo.packageList.Len())
seen := make(map[string]int, repo.packageList.Len())

55
deb/remote_test.go
View File

@@ -281,7 +281,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
c.Assert(err, IsNil)
c.Assert(s.downloader.Empty(), Equals, true)
queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(3))
c.Check(queue, HasLen, 1)
@@ -308,7 +308,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
c.Assert(err, IsNil)
c.Assert(s.downloader.Empty(), Equals, true)
queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(0))
c.Check(queue, HasLen, 0)
@@ -329,7 +329,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
c.Assert(err, IsNil)
c.Assert(s.downloader.Empty(), Equals, true)
queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(3))
c.Check(queue, HasLen, 1)
@@ -356,7 +356,7 @@ func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) {
c.Assert(err, IsNil)
c.Assert(s.downloader.Empty(), Equals, true)
queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(3)+int64(len(exampleInstallerManifestFile)))
c.Check(queue, HasLen, 2)
@@ -382,6 +382,35 @@ func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) {
c.Check(pkg.Name, Equals, "installer")
}
func (s *RemoteRepoSuite) TestBuildDownloadQueueLatestOnly(c *C) {
s.repo.Architectures = []string{"i386"}
err := s.repo.Fetch(s.downloader, nil, true)
c.Assert(err, IsNil)
s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, true, false)
c.Assert(err, IsNil)
c.Assert(s.downloader.Empty(), Equals, true)
stanza := packageStanza.Copy()
stanza["Package"] = "amanda-client"
stanza["Version"] = "1:3.4.0-1"
stanza["Filename"] = "pool/main/a/amanda/amanda-client_3.4.0-1_i386.deb"
newest := NewPackageFromControlFile(stanza)
_ = s.repo.packageList.Add(newest)
queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, true)
c.Assert(err, IsNil)
c.Check(queue, HasLen, 1)
c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.4.0-1_i386.deb")
c.Check(size, Equals, int64(187518))
}
func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
s.repo.Architectures = []string{"i386"}
s.repo.DownloadSources = true
@@ -400,7 +429,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
c.Assert(err, IsNil)
c.Assert(s.downloader.Empty(), Equals, true)
queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(15))
c.Check(queue, HasLen, 4)
@@ -444,7 +473,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
c.Assert(err, IsNil)
c.Assert(s.downloader.Empty(), Equals, true)
queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(0))
c.Check(queue, HasLen, 0)
@@ -469,7 +498,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
c.Assert(err, IsNil)
c.Assert(s.downloader.Empty(), Equals, true)
queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(15))
c.Check(queue, HasLen, 4)
@@ -493,7 +522,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
c.Assert(err, IsNil)
c.Assert(downloader.Empty(), Equals, true)
queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(3))
c.Check(queue, HasLen, 1)
@@ -521,7 +550,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
c.Assert(err, IsNil)
c.Assert(downloader.Empty(), Equals, true)
queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(0))
c.Check(queue, HasLen, 0)
@@ -543,7 +572,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
c.Assert(err, IsNil)
c.Assert(downloader.Empty(), Equals, true)
queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(3))
c.Check(queue, HasLen, 1)
@@ -574,7 +603,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
c.Assert(err, IsNil)
c.Assert(downloader.Empty(), Equals, true)
queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(15))
c.Check(queue, HasLen, 4)
@@ -620,7 +649,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
c.Assert(err, IsNil)
c.Assert(downloader.Empty(), Equals, true)
queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(0))
c.Check(queue, HasLen, 0)
@@ -646,7 +675,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
c.Assert(err, IsNil)
c.Assert(downloader.Empty(), Equals, true)
queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(15))
c.Check(queue, HasLen, 4)

4
man/aptly.1
View File

@@ -738,6 +738,10 @@ max download tries till process fails with download error
\-\fBskip\-existing\-packages\fR
do not check file existence for packages listed in the internal database of the mirror
.
.TP
\-\fBlatest\fR
download only latest version of each package (per architecture)
.
.SH "RENAMES MIRROR"
\fBaptly\fR \fBmirror\fR \fBrename\fR \fIold\-name\fR \fInew\-name\fR
.