mirror of
https://github.com/aptly-dev/aptly.git
synced 2026-01-12 03:21:33 +00:00
improve and test grab downloader
This commit is contained in:
André Roth
@@ -59,14 +59,13 @@ func (d *GrabDownloader) DownloadWithChecksum(ctx context.Context, url string, d
|
||||
// Success
|
||||
break
|
||||
}
|
||||
d.log("Download Error: %v\n", err)
|
||||
if retryableError(err) {
|
||||
maxTries--
|
||||
d.log("Retrying download %s: %d\n", url, maxTries)
|
||||
d.log("Retrying %d %s\n", maxTries, url)
|
||||
time.Sleep(delay)
|
||||
} else {
|
||||
// Can't retry
|
||||
d.log("Cannot retry download %s\n", url)
|
||||
d.log("Error (retrying): HTTP code %s while fetching %s\n", err, url)
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -115,8 +114,7 @@ func (d *GrabDownloader) maybeSetupChecksum(req *grab.Request, expected *utils.C
|
||||
}
|
||||
|
||||
func (d *GrabDownloader) download(_ context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error {
|
||||
// TODO clean up dest dir on permanent failure
|
||||
d.log("Download %s -> %s\n", url, destination)
|
||||
d.log("Downloading: %s\n", url)
|
||||
|
||||
req, err := grab.NewRequest(destination, url)
|
||||
if err != nil {
|
||||
|
||||
33
system/t04_mirror/UpdateMirror26Test_gold
Normal file
33
system/t04_mirror/UpdateMirror26Test_gold
Normal file
@@ -0,0 +1,33 @@
|
||||
|
||||
|
||||
Applying filter...
|
||||
Building download queue...
|
||||
Download queue: 1 items (3.35 KiB)
|
||||
Downloading & parsing package files...
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb
|
||||
Mirror `grab` has been successfully updated.
|
||||
Packages filtered: 50604 -> 1.
|
||||
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||
gpgv: issuer "debian-release@lists.debian.org"
|
||||
gpgv: using RSA key 0146DC6D4A0B2914BDED34DB648ACFD622F3D138
|
||||
gpgv: using RSA key 067E3C456BAE240ACEE88F6FEF0F382A1A7B6500
|
||||
gpgv: using RSA key 16E90B3FDF65EDE3AA7F323C04EE7237B7D453EC
|
||||
gpgv: using RSA key A7236886F3CCCAAD148A27F80E98404D386FA1D9
|
||||
gpgv: Good signature from "Debian Archive Automatic Signing Key (10/buster) <ftpmaster@debian.org>"
|
||||
gpgv: Good signature from "Debian Archive Automatic Signing Key (11/bullseye) <ftpmaster@debian.org>"
|
||||
gpgv: Good signature from "Debian Archive Automatic Signing Key (9/stretch) <ftpmaster@debian.org>"
|
||||
gpgv: Good signature from "Debian Stable Release Key (9/stretch) <debian-release@lists.debian.org>"
|
||||
gpgv: Signature made Sat Aug 14 07:43:24 2021 UTC
|
||||
gpgv: Signature made Sat Aug 14 07:43:25 2021 UTC
|
||||
gpgv: Signature made Sat Aug 14 08:26:43 2021 UTC
|
||||
gpgv: Signature made Sat Aug 14 08:46:19 2021 UTC
|
||||
39
system/t04_mirror/UpdateMirror27Test_gold
Normal file
39
system/t04_mirror/UpdateMirror27Test_gold
Normal file
@@ -0,0 +1,39 @@
|
||||
|
||||
server returned 404 Not Found
|
||||
Applying filter...
|
||||
Building download queue...
|
||||
Download queue: 2 items (6.70 KiB)
|
||||
Downloading & parsing package files...
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-amd64/Packages.gz
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-amd64/Packages.gz
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb
|
||||
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb
|
||||
ERROR: unable to update: download errors:
|
||||
Packages filtered: 76844 -> 2.
|
||||
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb
|
||||
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb
|
||||
gpgv: issuer "debian-release@lists.debian.org"
|
||||
gpgv: using RSA key 0146DC6D4A0B2914BDED34DB648ACFD622F3D138
|
||||
gpgv: using RSA key 067E3C456BAE240ACEE88F6FEF0F382A1A7B6500
|
||||
gpgv: using RSA key 16E90B3FDF65EDE3AA7F323C04EE7237B7D453EC
|
||||
gpgv: using RSA key A7236886F3CCCAAD148A27F80E98404D386FA1D9
|
||||
gpgv: Good signature from "Debian Archive Automatic Signing Key (10/buster) <ftpmaster@debian.org>"
|
||||
gpgv: Good signature from "Debian Archive Automatic Signing Key (11/bullseye) <ftpmaster@debian.org>"
|
||||
gpgv: Good signature from "Debian Archive Automatic Signing Key (9/stretch) <ftpmaster@debian.org>"
|
||||
gpgv: Good signature from "Debian Stable Release Key (9/stretch) <debian-release@lists.debian.org>"
|
||||
gpgv: Signature made Sat Aug 14 07:43:24 2021 UTC
|
||||
gpgv: Signature made Sat Aug 14 07:43:25 2021 UTC
|
||||
gpgv: Signature made Sat Aug 14 08:26:43 2021 UTC
|
||||
gpgv: Signature made Sat Aug 14 08:46:19 2021 UTC
|
||||
@@ -451,3 +451,34 @@ class UpdateMirror25Test(BaseTest):
|
||||
]
|
||||
runCmd = "aptly mirror update -keyring=aptlytest.gpg mirror19"
|
||||
outputMatchPrepare = filterOutSignature
|
||||
|
||||
|
||||
class UpdateMirror26Test(BaseTest):
|
||||
"""
|
||||
update mirrors: regular update, grab downloader
|
||||
"""
|
||||
configOverride = {"downloader": "grab"}
|
||||
sortOutput = True
|
||||
longTest = False
|
||||
fixtureGpg = True
|
||||
fixtureCmds = [
|
||||
"aptly mirror create -architectures=i386 -keyring=aptlytest.gpg -filter=libboost-program-options-dev grab http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian stretch main",
|
||||
]
|
||||
runCmd = "aptly mirror update -downloader=grab -keyring=aptlytest.gpg grab"
|
||||
outputMatchPrepare = filterOutRedirects
|
||||
|
||||
|
||||
class UpdateMirror27Test(BaseTest):
|
||||
"""
|
||||
update mirrors: failing update, grab downloader
|
||||
"""
|
||||
configOverride = {"downloader": "grab"}
|
||||
sortOutput = True
|
||||
longTest = False
|
||||
fixtureGpg = True
|
||||
fixtureCmds = [
|
||||
"aptly mirror create -architectures=amd64,i386 -keyring=aptlytest.gpg -filter=libboost-program-options-dev grab-fail http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian stretch main",
|
||||
]
|
||||
runCmd = "aptly mirror update -downloader=grab -keyring=aptlytest.gpg grab-fail"
|
||||
outputMatchPrepare = filterOutRedirects
|
||||
expectedCode = 1
|
||||
|
||||
Reference in New Issue
Block a user