From 32b601bde629f1fd61fcb4f7b401cec12f2d083d Mon Sep 17 00:00:00 2001 From: Linus Fischer Date: Tue, 2 Dec 2025 11:03:28 +0000 Subject: [PATCH 1/5] Fix swagger property casing --- .github/workflows/golangci-lint.yml | 2 +- Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index dad2bdc5..6bc15fbb 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -35,7 +35,7 @@ jobs: - name: Install and initialize swagger run: | go install github.com/swaggo/swag/cmd/swag@latest - swag init -q --markdownFiles docs + swag init -q --propertyStrategy pascalcase --markdownFiles docs shell: sh - name: golangci-lint diff --git a/Makefile b/Makefile index c8b7628d..dc1973d1 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ azurite-stop: swagger: swagger-install # Generate swagger docs - @PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf + @PATH=$(BINPATH)/:$(PATH) swag init --propertyStrategy pascalcase --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf etcd-install: # Install etcd @@ -131,7 +131,7 @@ serve: prepare swagger-install ## Run development server (auto recompiling) test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3 cp debian/aptly.conf ~/.aptly.conf sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf - PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142 + PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --propertyStrategy pascalcase --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142 dpkg: prepare swagger ## Build debian packages @test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1) From 0021cf876b558c2824495f0076c54b32a16b3c64 Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez <835733+gaby@users.noreply.github.com> Date: Thu, 11 Dec 2025 07:20:37 -0500 Subject: [PATCH 2/5] Harden latest-only filtering --- api/mirror.go | 4 +++- cmd/mirror_update.go | 4 +++- completion.d/_aptly | 1 + completion.d/aptly | 2 +- deb/list.go | 33 ++++++++++++++++++++++++++ deb/list_test.go | 49 +++++++++++++++++++++++++++++++++++++++ deb/remote.go | 14 ++++++++++- deb/remote_test.go | 55 +++++++++++++++++++++++++++++++++----------- man/aptly.1 | 4 ++++ 9 files changed, 149 insertions(+), 17 deletions(-) diff --git a/api/mirror.go b/api/mirror.go index 77de795c..60c8f23c 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -343,6 +343,8 @@ type mirrorUpdateParams struct { ForceUpdate bool ` json:"ForceUpdate"` // Set "true" to skip downloading already downloaded packages SkipExistingPackages bool ` json:"SkipExistingPackages"` + // Set "true" to download only the latest version per package/architecture + LatestOnly bool ` json:"LatestOnly"` } // @Summary Update Mirror @@ -434,7 +436,7 @@ func apiMirrorsUpdate(c *gin.Context) { } queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(), - collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages) + collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages, b.LatestOnly) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } diff --git a/cmd/mirror_update.go b/cmd/mirror_update.go index 2f0fb524..df28efa7 100644 --- a/cmd/mirror_update.go +++ b/cmd/mirror_update.go @@ -87,10 +87,11 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { ) skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool) + latestOnly := context.Flags().Lookup("latest").Value.Get().(bool) context.Progress().Printf("Building download queue...\n") queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(), - collectionFactory.ChecksumCollection(nil), skipExistingPackages) + collectionFactory.ChecksumCollection(nil), skipExistingPackages, latestOnly) if err != nil { return fmt.Errorf("unable to update: %s", err) @@ -292,6 +293,7 @@ Example: cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata") cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror") + cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture)") cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)") cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)") cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error") diff --git a/completion.d/_aptly b/completion.d/_aptly index 8e8d2bbd..a1f4d859 100644 --- a/completion.d/_aptly +++ b/completion.d/_aptly @@ -211,6 +211,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could $keyring \ "-max-tries=[max download tries till process fails with download error]:number: " \ "-skip-existing-packages=[do not check file existence for packages listed in the internal database of the mirror]:$bool" \ + "-latest=[download only latest version of each package (per architecture)]:$bool" \ "(-)2:mirror name:$mirrors" ;; rename) diff --git a/completion.d/aptly b/completion.d/aptly index ab8c4c15..aa5b506f 100644 --- a/completion.d/aptly +++ b/completion.d/aptly @@ -263,7 +263,7 @@ _aptly() "update") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur})) + COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages -latest" -- ${cur})) else COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur})) fi diff --git a/deb/list.go b/deb/list.go index 25a2d283..19e37593 100644 --- a/deb/list.go +++ b/deb/list.go @@ -172,6 +172,39 @@ func (l *PackageList) ForEach(handler func(*Package) error) error { return err } +// FilterLatest creates a copy of the package list containing only the +// latest version for each package name/architecture pair. +func (l *PackageList) FilterLatest() (*PackageList, error) { + if l == nil { + return nil, fmt.Errorf("package list is nil") + } + + filtered := make(map[string]*Package, l.Len()) + + err := l.ForEach(func(p *Package) error { + key := p.Architecture + "|" + p.Name + + if existing, found := filtered[key]; !found || CompareVersions(p.Version, existing.Version) > 0 { + filtered[key] = p + } + + return nil + }) + if err != nil { + return nil, err + } + + result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(filtered)) + + for _, pkg := range filtered { + if err = result.Add(pkg); err != nil { + return nil, err + } + } + + return result, nil +} + // ForEachIndexed calls handler for each package in list in indexed order func (l *PackageList) ForEachIndexed(handler func(*Package) error) error { if !l.indexed { diff --git a/deb/list_test.go b/deb/list_test.go index 464dd09e..4e4768e3 100644 --- a/deb/list_test.go +++ b/deb/list_test.go @@ -503,3 +503,52 @@ func (s *PackageListSuite) TestArchitectures(c *C) { sort.Strings(archs) c.Check(archs, DeepEquals, []string{"amd64", "arm", "i386", "s390"}) } + +func (s *PackageListSuite) TestFilterLatest(c *C) { + list := NewPackageList() + + older := packageStanza.Copy() + older["Version"] = "1.0" + olderPkg := NewPackageFromControlFile(older) + _ = list.Add(olderPkg) + + newer := packageStanza.Copy() + newer["Version"] = "2.0" + newerPkg := NewPackageFromControlFile(newer) + _ = list.Add(newerPkg) + + shared := packageStanza.Copy() + shared["Architecture"] = ArchitectureAll + shared["Version"] = "3.0" + shared["Package"] = "shared" + sharedPkg := NewPackageFromControlFile(shared) + _ = list.Add(sharedPkg) + + filtered, err := list.FilterLatest() + c.Assert(err, IsNil) + c.Assert(filtered.Len(), Equals, 2) + c.Check(filtered.Has(newerPkg), Equals, true) + c.Check(filtered.Has(sharedPkg), Equals, true) +} + +func (s *PackageListSuite) TestFilterLatestPreservesDuplicatesFlag(c *C) { + list := NewPackageListWithDuplicates(true, 2) + + _ = list.Add(NewPackageFromControlFile(packageStanza.Copy())) + + another := packageStanza.Copy() + another["Version"] = "7.41-1" + _ = list.Add(NewPackageFromControlFile(another)) + + filtered, err := list.FilterLatest() + c.Assert(err, IsNil) + c.Assert(filtered.duplicatesAllowed, Equals, true) +} + +func (s *PackageListSuite) TestFilterLatestNil(c *C) { + var list *PackageList + + filtered, err := list.FilterLatest() + c.Assert(err, ErrorMatches, "package list is nil") + c.Assert(filtered, IsNil) +} diff --git a/deb/remote.go b/deb/remote.go index c6fe595c..8776b65a 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -612,7 +612,19 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu } // BuildDownloadQueue builds queue, discards current PackageList -func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) { +func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages, latestOnly bool) (queue []PackageDownloadTask, downloadSize int64, err error) { + if repo.packageList == nil { + err = fmt.Errorf("package list is empty, please (re)download package indexes") + return + } + + if latestOnly { + repo.packageList, err = repo.packageList.FilterLatest() + if err != nil { + return + } + } + queue = make([]PackageDownloadTask, 0, repo.packageList.Len()) seen := make(map[string]int, repo.packageList.Len()) diff --git a/deb/remote_test.go b/deb/remote_test.go index c0eae7f5..88168585 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -281,7 +281,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -308,7 +308,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -329,7 +329,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -356,7 +356,7 @@ func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)+int64(len(exampleInstallerManifestFile))) c.Check(queue, HasLen, 2) @@ -382,6 +382,35 @@ func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) { c.Check(pkg.Name, Equals, "installer") } +func (s *RemoteRepoSuite) TestBuildDownloadQueueLatestOnly(c *C) { + s.repo.Architectures = []string{"i386"} + + err := s.repo.Fetch(s.downloader, nil, true) + c.Assert(err, IsNil) + + s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404}) + s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404}) + s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile) + + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, true, false) + c.Assert(err, IsNil) + c.Assert(s.downloader.Empty(), Equals, true) + + stanza := packageStanza.Copy() + stanza["Package"] = "amanda-client" + stanza["Version"] = "1:3.4.0-1" + stanza["Filename"] = "pool/main/a/amanda/amanda-client_3.4.0-1_i386.deb" + + newest := NewPackageFromControlFile(stanza) + _ = s.repo.packageList.Add(newest) + + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, true) + c.Assert(err, IsNil) + c.Check(queue, HasLen, 1) + c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.4.0-1_i386.deb") + c.Check(size, Equals, int64(187518)) +} + func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { s.repo.Architectures = []string{"i386"} s.repo.DownloadSources = true @@ -400,7 +429,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) @@ -444,7 +473,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -469,7 +498,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) @@ -493,7 +522,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -521,7 +550,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -543,7 +572,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -574,7 +603,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) @@ -620,7 +649,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -646,7 +675,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) diff --git a/man/aptly.1 b/man/aptly.1 index bd6ad223..f1ab1eee 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -738,6 +738,10 @@ max download tries till process fails with download error \-\fBskip\-existing\-packages\fR do not check file existence for packages listed in the internal database of the mirror . +.TP +\-\fBlatest\fR +download only latest version of each package (per architecture) +. .SH "RENAMES MIRROR" \fBaptly\fR \fBmirror\fR \fBrename\fR \fIold\-name\fR \fInew\-name\fR . From de699aebe55262fc69738846b5909e9a30b42ab3 Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez <835733+gaby@users.noreply.github.com> Date: Thu, 11 Dec 2025 09:42:15 -0500 Subject: [PATCH 3/5] Update AUTHORS list --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 2c067dd2..4f5efee0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -74,3 +74,4 @@ List of contributors, in chronological order: * JupiterRider (https://github.com/JupiterRider) * Agustin Henze (https://github.com/agustinhenze) * Tobias Assarsson (https://github.com/daedaluz) +* Juan Calderon-Perez (https://github.com/gaby) From af483d1165440cc8637071430bf1c5391d11bf0d Mon Sep 17 00:00:00 2001 From: Ato Araki Date: Thu, 13 Nov 2025 14:46:27 +0900 Subject: [PATCH 4/5] feat(http): add GCP authentication for ar+https scheme --- go.mod | 4 +- go.sum | 4 ++ http/download.go | 1 + http/gcp_auth.go | 64 ++++++++++++++++++++++++ http/gcp_auth_test.go | 110 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 http/gcp_auth.go create mode 100644 http/gcp_auth_test.go diff --git a/go.mod b/go.mod index 53c5e78c..6ef36305 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/aptly-dev/aptly -go 1.24 +go 1.24.0 require ( github.com/AlekSi/pointer v1.1.0 @@ -41,6 +41,7 @@ require ( ) require ( + cloud.google.com/go/compute/metadata v0.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect @@ -128,5 +129,6 @@ require ( github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.3 go.etcd.io/etcd/client/v3 v3.5.15 + golang.org/x/oauth2 v0.33.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 502f4b21..e53d552a 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= github.com/AlekSi/pointer v1.1.0 h1:SSDMPcXD9jSl8FPy9cRzoRaMJtm9g9ggGTxecRUbQoI= github.com/AlekSi/pointer v1.1.0/go.mod h1:y7BvfRI3wXPWKXEBhU71nbnIEEZX0QTSB2Bj48UJIZE= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= @@ -348,6 +350,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= +golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/http/download.go b/http/download.go index 887f9b3b..02b9275e 100644 --- a/http/download.go +++ b/http/download.go @@ -44,6 +44,7 @@ func NewDownloader(downLimit int64, maxTries int, progress aptly.Progress) aptly transport.DisableCompression = true initTransport(&transport) transport.RegisterProtocol("ftp", &protocol.FTPRoundTripper{}) + transport.RegisterProtocol("ar+https", NewGCPRoundTripper(&transport)) downloader := &downloaderImpl{ progress: progress, diff --git a/http/gcp_auth.go b/http/gcp_auth.go new file mode 100644 index 00000000..d547da53 --- /dev/null +++ b/http/gcp_auth.go @@ -0,0 +1,64 @@ +package http + +import ( + "context" + "fmt" + "net/http" + "strings" + "sync" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" +) + +// gcpRoundTripper wraps http.RoundTripper to add Google Cloud authentication. +// It delays GCP authentication initialization until the first actual request is made. +// This avoids unnecessary credential loading when ar+https protocol is not actually used. +// +// It uses Application Default Credentials (ADC) which checks: +// 1. GOOGLE_APPLICATION_CREDENTIALS environment variable +// 2. gcloud auth application-default credentials +// 3. GCE/GKE metadata server +// See https://cloud.google.com/docs/authentication/application-default-credentials for usage details. +type gcpRoundTripper struct { + base http.RoundTripper + initOnce sync.Once + tokenSrc oauth2.TokenSource + initErr error +} + +func (t *gcpRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // Lazy initialization: only initialize GCP credentials on first request + t.initOnce.Do(func() { + creds, err := google.FindDefaultCredentials(context.Background(), + "https://www.googleapis.com/auth/cloud-platform") + if err != nil { + t.initErr = fmt.Errorf("failed to find default credentials: %w", err) + return + } + t.tokenSrc = creds.TokenSource + }) + + reqCopy := req.Clone(req.Context()) + reqCopy.URL.Scheme = strings.TrimPrefix(reqCopy.URL.Scheme, "ar+") + + // Fall back to base transport if GCP auth initialization failed + if t.initErr != nil { + return t.base.RoundTrip(reqCopy) + } + + token, err := t.tokenSrc.Token() + if err != nil { + return nil, fmt.Errorf("failed to get OAuth2 token: %w", err) + } + token.SetAuthHeader(reqCopy) + + return t.base.RoundTrip(reqCopy) +} + +// NewGCPRoundTripper creates a new RoundTripper that handles GCP authentication for ar+https protocol. +func NewGCPRoundTripper(base http.RoundTripper) http.RoundTripper { + return &gcpRoundTripper{ + base: base, + } +} diff --git a/http/gcp_auth_test.go b/http/gcp_auth_test.go new file mode 100644 index 00000000..a706bdfe --- /dev/null +++ b/http/gcp_auth_test.go @@ -0,0 +1,110 @@ +package http + +import ( + "net/http" + "net/http/httptest" + "os" + "testing" + + "golang.org/x/oauth2" +) + +func TestGCPAuthTransport_RoundTrip(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + auth := r.Header.Get("Authorization") + if auth == "" { + t.Error("Expected Authorization header, got none") + } + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + transport := NewGCPRoundTripper(http.DefaultTransport) + + if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { + t.Skip("Skipping test: GOOGLE_APPLICATION_CREDENTIALS not set") + } + + client := &http.Client{Transport: transport} + resp, err := client.Get(ts.URL) + if err != nil { + t.Fatalf("Failed to make request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } +} + +func TestGCPAuthTransport_RoundTrip_with_dummy_tokenSource(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + auth := r.Header.Get("Authorization") + if auth != "Bearer dummy-token" { + t.Errorf("Expected Authorization header 'Bearer dummy-token', got '%s'", auth) + } + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + // Use a dummy token source for testing + transport := &gcpRoundTripper{ + base: http.DefaultTransport, + tokenSrc: oauth2.StaticTokenSource(&oauth2.Token{ + AccessToken: "dummy-token", + }), + } + transport.initOnce.Do(func() {}) // Mark as initialized for testing + + client := &http.Client{Transport: transport} + resp, err := client.Get(ts.URL) + if err != nil { + t.Fatalf("Failed to make request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } +} + +func TestGCPAuthTransport_RoundTrip_with_InvalidCredentials(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer ts.Close() + + // Create a temporary invalid credentials file + tmpFile, err := os.CreateTemp("", "invalid_credentials.json") + if err != nil { + t.Fatalf("Failed to create temp file: %s", err) + } + defer os.Remove(tmpFile.Name()) + if _, err := tmpFile.WriteString(`{"invalid": "data"}`); err != nil { + t.Fatalf("Failed to write to temp file: %s", err) + } + tmpFile.Close() + + defaultEnv := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", tmpFile.Name()) + defer os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", defaultEnv) + + transport := &gcpRoundTripper{ + base: http.DefaultTransport, + } + + client := &http.Client{Transport: transport} + resp, err := client.Get(ts.URL) + if err != nil { + t.Fatalf("Failed to make request: %s", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusForbidden { + t.Errorf("Expected status 403, got %d", resp.StatusCode) + } + + if transport.initErr == nil { + t.Error("Expected init error due to invalid credentials, got none") + } +} From bcd81eeae4a36be4bc5bcaa32f288dad115a6365 Mon Sep 17 00:00:00 2001 From: Ato Araki Date: Fri, 14 Nov 2025 10:26:45 +0900 Subject: [PATCH 5/5] update AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 2c067dd2..11d04e33 100644 --- a/AUTHORS +++ b/AUTHORS @@ -74,3 +74,4 @@ List of contributors, in chronological order: * JupiterRider (https://github.com/JupiterRider) * Agustin Henze (https://github.com/agustinhenze) * Tobias Assarsson (https://github.com/daedaluz) +* Ato Araki (https://github.com/atotto)