From f0ad0f949641de619d6f0d5bd0c20ec5ed10dcd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 4 Oct 2024 12:26:44 +0200 Subject: [PATCH] improve and test grab downloader --- http/grab.go | 8 ++--- system/t04_mirror/UpdateMirror26Test_gold | 33 +++++++++++++++++++ system/t04_mirror/UpdateMirror27Test_gold | 39 +++++++++++++++++++++++ system/t04_mirror/update.py | 31 ++++++++++++++++++ 4 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 system/t04_mirror/UpdateMirror26Test_gold create mode 100644 system/t04_mirror/UpdateMirror27Test_gold diff --git a/http/grab.go b/http/grab.go index 0bad19c5..51fc672b 100644 --- a/http/grab.go +++ b/http/grab.go @@ -59,14 +59,13 @@ func (d *GrabDownloader) DownloadWithChecksum(ctx context.Context, url string, d // Success break } - d.log("Download Error: %v\n", err) if retryableError(err) { maxTries-- - d.log("Retrying download %s: %d\n", url, maxTries) + d.log("Retrying %d %s\n", maxTries, url) time.Sleep(delay) } else { // Can't retry - d.log("Cannot retry download %s\n", url) + d.log("Error (retrying): HTTP code %s while fetching %s\n", err, url) break } } @@ -115,8 +114,7 @@ func (d *GrabDownloader) maybeSetupChecksum(req *grab.Request, expected *utils.C } func (d *GrabDownloader) download(_ context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error { - // TODO clean up dest dir on permanent failure - d.log("Download %s -> %s\n", url, destination) + d.log("Downloading: %s\n", url) req, err := grab.NewRequest(destination, url) if err != nil { diff --git a/system/t04_mirror/UpdateMirror26Test_gold b/system/t04_mirror/UpdateMirror26Test_gold new file mode 100644 index 00000000..aa6d212e --- /dev/null +++ b/system/t04_mirror/UpdateMirror26Test_gold @@ -0,0 +1,33 @@ + + +Applying filter... +Building download queue... +Download queue: 1 items (3.35 KiB) +Downloading & parsing package files... +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb +Mirror `grab` has been successfully updated. +Packages filtered: 50604 -> 1. +Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease +Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease +gpgv: issuer "debian-release@lists.debian.org" +gpgv: using RSA key 0146DC6D4A0B2914BDED34DB648ACFD622F3D138 +gpgv: using RSA key 067E3C456BAE240ACEE88F6FEF0F382A1A7B6500 +gpgv: using RSA key 16E90B3FDF65EDE3AA7F323C04EE7237B7D453EC +gpgv: using RSA key A7236886F3CCCAAD148A27F80E98404D386FA1D9 +gpgv: Good signature from "Debian Archive Automatic Signing Key (10/buster) " +gpgv: Good signature from "Debian Archive Automatic Signing Key (11/bullseye) " +gpgv: Good signature from "Debian Archive Automatic Signing Key (9/stretch) " +gpgv: Good signature from "Debian Stable Release Key (9/stretch) " +gpgv: Signature made Sat Aug 14 07:43:24 2021 UTC +gpgv: Signature made Sat Aug 14 07:43:25 2021 UTC +gpgv: Signature made Sat Aug 14 08:26:43 2021 UTC +gpgv: Signature made Sat Aug 14 08:46:19 2021 UTC \ No newline at end of file diff --git a/system/t04_mirror/UpdateMirror27Test_gold b/system/t04_mirror/UpdateMirror27Test_gold new file mode 100644 index 00000000..677e80cb --- /dev/null +++ b/system/t04_mirror/UpdateMirror27Test_gold @@ -0,0 +1,39 @@ + + server returned 404 Not Found +Applying filter... +Building download queue... +Download queue: 2 items (6.70 KiB) +Downloading & parsing package files... +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-amd64/Packages.gz +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-amd64/Packages.gz +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb +Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb +ERROR: unable to update: download errors: +Packages filtered: 76844 -> 2. +Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease +Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease +Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb +Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb +gpgv: issuer "debian-release@lists.debian.org" +gpgv: using RSA key 0146DC6D4A0B2914BDED34DB648ACFD622F3D138 +gpgv: using RSA key 067E3C456BAE240ACEE88F6FEF0F382A1A7B6500 +gpgv: using RSA key 16E90B3FDF65EDE3AA7F323C04EE7237B7D453EC +gpgv: using RSA key A7236886F3CCCAAD148A27F80E98404D386FA1D9 +gpgv: Good signature from "Debian Archive Automatic Signing Key (10/buster) " +gpgv: Good signature from "Debian Archive Automatic Signing Key (11/bullseye) " +gpgv: Good signature from "Debian Archive Automatic Signing Key (9/stretch) " +gpgv: Good signature from "Debian Stable Release Key (9/stretch) " +gpgv: Signature made Sat Aug 14 07:43:24 2021 UTC +gpgv: Signature made Sat Aug 14 07:43:25 2021 UTC +gpgv: Signature made Sat Aug 14 08:26:43 2021 UTC +gpgv: Signature made Sat Aug 14 08:46:19 2021 UTC \ No newline at end of file diff --git a/system/t04_mirror/update.py b/system/t04_mirror/update.py index e950d981..fb44dd7e 100644 --- a/system/t04_mirror/update.py +++ b/system/t04_mirror/update.py @@ -451,3 +451,34 @@ class UpdateMirror25Test(BaseTest): ] runCmd = "aptly mirror update -keyring=aptlytest.gpg mirror19" outputMatchPrepare = filterOutSignature + + +class UpdateMirror26Test(BaseTest): + """ + update mirrors: regular update, grab downloader + """ + configOverride = {"downloader": "grab"} + sortOutput = True + longTest = False + fixtureGpg = True + fixtureCmds = [ + "aptly mirror create -architectures=i386 -keyring=aptlytest.gpg -filter=libboost-program-options-dev grab http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian stretch main", + ] + runCmd = "aptly mirror update -downloader=grab -keyring=aptlytest.gpg grab" + outputMatchPrepare = filterOutRedirects + + +class UpdateMirror27Test(BaseTest): + """ + update mirrors: failing update, grab downloader + """ + configOverride = {"downloader": "grab"} + sortOutput = True + longTest = False + fixtureGpg = True + fixtureCmds = [ + "aptly mirror create -architectures=amd64,i386 -keyring=aptlytest.gpg -filter=libboost-program-options-dev grab-fail http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian stretch main", + ] + runCmd = "aptly mirror update -downloader=grab -keyring=aptlytest.gpg grab-fail" + outputMatchPrepare = filterOutRedirects + expectedCode = 1