mirror of
https://github.com/aptly-dev/aptly.git
synced 2026-07-25 13:37:51 +00:00
improve and test grab downloader
This commit is contained in:
André Roth
+3
-5
@@ -59,14 +59,13 @@ func (d *GrabDownloader) DownloadWithChecksum(ctx context.Context, url string, d
|
|||||||
// Success
|
// Success
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
d.log("Download Error: %v\n", err)
|
|
||||||
if retryableError(err) {
|
if retryableError(err) {
|
||||||
maxTries--
|
maxTries--
|
||||||
d.log("Retrying download %s: %d\n", url, maxTries)
|
d.log("Retrying %d %s\n", maxTries, url)
|
||||||
time.Sleep(delay)
|
time.Sleep(delay)
|
||||||
} else {
|
} else {
|
||||||
// Can't retry
|
// Can't retry
|
||||||
d.log("Cannot retry download %s\n", url)
|
d.log("Error (retrying): HTTP code %s while fetching %s\n", err, url)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -115,8 +114,7 @@ func (d *GrabDownloader) maybeSetupChecksum(req *grab.Request, expected *utils.C
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *GrabDownloader) download(_ context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error {
|
func (d *GrabDownloader) download(_ context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error {
|
||||||
// TODO clean up dest dir on permanent failure
|
d.log("Downloading: %s\n", url)
|
||||||
d.log("Download %s -> %s\n", url, destination)
|
|
||||||
|
|
||||||
req, err := grab.NewRequest(destination, url)
|
req, err := grab.NewRequest(destination, url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
|
||||||
|
|
||||||
|
Applying filter...
|
||||||
|
Building download queue...
|
||||||
|
Download queue: 1 items (3.35 KiB)
|
||||||
|
Downloading & parsing package files...
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb
|
||||||
|
Mirror `grab` has been successfully updated.
|
||||||
|
Packages filtered: 50604 -> 1.
|
||||||
|
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||||
|
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||||
|
gpgv: issuer "debian-release@lists.debian.org"
|
||||||
|
gpgv: using RSA key 0146DC6D4A0B2914BDED34DB648ACFD622F3D138
|
||||||
|
gpgv: using RSA key 067E3C456BAE240ACEE88F6FEF0F382A1A7B6500
|
||||||
|
gpgv: using RSA key 16E90B3FDF65EDE3AA7F323C04EE7237B7D453EC
|
||||||
|
gpgv: using RSA key A7236886F3CCCAAD148A27F80E98404D386FA1D9
|
||||||
|
gpgv: Good signature from "Debian Archive Automatic Signing Key (10/buster) <ftpmaster@debian.org>"
|
||||||
|
gpgv: Good signature from "Debian Archive Automatic Signing Key (11/bullseye) <ftpmaster@debian.org>"
|
||||||
|
gpgv: Good signature from "Debian Archive Automatic Signing Key (9/stretch) <ftpmaster@debian.org>"
|
||||||
|
gpgv: Good signature from "Debian Stable Release Key (9/stretch) <debian-release@lists.debian.org>"
|
||||||
|
gpgv: Signature made Sat Aug 14 07:43:24 2021 UTC
|
||||||
|
gpgv: Signature made Sat Aug 14 07:43:25 2021 UTC
|
||||||
|
gpgv: Signature made Sat Aug 14 08:26:43 2021 UTC
|
||||||
|
gpgv: Signature made Sat Aug 14 08:46:19 2021 UTC
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
|
||||||
|
server returned 404 Not Found
|
||||||
|
Applying filter...
|
||||||
|
Building download queue...
|
||||||
|
Download queue: 2 items (6.70 KiB)
|
||||||
|
Downloading & parsing package files...
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-amd64/Packages.gz
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-amd64/Packages.gz
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb
|
||||||
|
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb
|
||||||
|
ERROR: unable to update: download errors:
|
||||||
|
Packages filtered: 76844 -> 2.
|
||||||
|
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||||
|
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
|
||||||
|
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb
|
||||||
|
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_amd64.deb
|
||||||
|
gpgv: issuer "debian-release@lists.debian.org"
|
||||||
|
gpgv: using RSA key 0146DC6D4A0B2914BDED34DB648ACFD622F3D138
|
||||||
|
gpgv: using RSA key 067E3C456BAE240ACEE88F6FEF0F382A1A7B6500
|
||||||
|
gpgv: using RSA key 16E90B3FDF65EDE3AA7F323C04EE7237B7D453EC
|
||||||
|
gpgv: using RSA key A7236886F3CCCAAD148A27F80E98404D386FA1D9
|
||||||
|
gpgv: Good signature from "Debian Archive Automatic Signing Key (10/buster) <ftpmaster@debian.org>"
|
||||||
|
gpgv: Good signature from "Debian Archive Automatic Signing Key (11/bullseye) <ftpmaster@debian.org>"
|
||||||
|
gpgv: Good signature from "Debian Archive Automatic Signing Key (9/stretch) <ftpmaster@debian.org>"
|
||||||
|
gpgv: Good signature from "Debian Stable Release Key (9/stretch) <debian-release@lists.debian.org>"
|
||||||
|
gpgv: Signature made Sat Aug 14 07:43:24 2021 UTC
|
||||||
|
gpgv: Signature made Sat Aug 14 07:43:25 2021 UTC
|
||||||
|
gpgv: Signature made Sat Aug 14 08:26:43 2021 UTC
|
||||||
|
gpgv: Signature made Sat Aug 14 08:46:19 2021 UTC
|
||||||
@@ -451,3 +451,34 @@ class UpdateMirror25Test(BaseTest):
|
|||||||
]
|
]
|
||||||
runCmd = "aptly mirror update -keyring=aptlytest.gpg mirror19"
|
runCmd = "aptly mirror update -keyring=aptlytest.gpg mirror19"
|
||||||
outputMatchPrepare = filterOutSignature
|
outputMatchPrepare = filterOutSignature
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateMirror26Test(BaseTest):
|
||||||
|
"""
|
||||||
|
update mirrors: regular update, grab downloader
|
||||||
|
"""
|
||||||
|
configOverride = {"downloader": "grab"}
|
||||||
|
sortOutput = True
|
||||||
|
longTest = False
|
||||||
|
fixtureGpg = True
|
||||||
|
fixtureCmds = [
|
||||||
|
"aptly mirror create -architectures=i386 -keyring=aptlytest.gpg -filter=libboost-program-options-dev grab http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian stretch main",
|
||||||
|
]
|
||||||
|
runCmd = "aptly mirror update -downloader=grab -keyring=aptlytest.gpg grab"
|
||||||
|
outputMatchPrepare = filterOutRedirects
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateMirror27Test(BaseTest):
|
||||||
|
"""
|
||||||
|
update mirrors: failing update, grab downloader
|
||||||
|
"""
|
||||||
|
configOverride = {"downloader": "grab"}
|
||||||
|
sortOutput = True
|
||||||
|
longTest = False
|
||||||
|
fixtureGpg = True
|
||||||
|
fixtureCmds = [
|
||||||
|
"aptly mirror create -architectures=amd64,i386 -keyring=aptlytest.gpg -filter=libboost-program-options-dev grab-fail http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian stretch main",
|
||||||
|
]
|
||||||
|
runCmd = "aptly mirror update -downloader=grab -keyring=aptlytest.gpg grab-fail"
|
||||||
|
outputMatchPrepare = filterOutRedirects
|
||||||
|
expectedCode = 1
|
||||||
|
|||||||
Reference in New Issue
Block a user