vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-06-07 05:42:42 +00:00
Code Issues Projects Releases Wiki Activity

Rework HTTP downloader retry logic

Browse Source 
Apply retries as global, config-level option `downloadRetries` so that
it can be applied to any aptly command which downloads objects.

Unwrap `errors.Wrap` which is used in downloader.

Unwrap `*url.Error` which should be the actual error returned from the
HTTP client, catch more cases, be more specific around failures.
This commit is contained in:
Andrey Smirnov
2019-08-06 00:41:14 +03:00
committed by Andrey Smirnov
parent 2e7f624b34
commit f0a370db24
22 changed files with 123 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
http/download.go
+36 -4
View File
@@ -6,6 +6,7 @@ import (
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
@@ -28,12 +29,13 @@ var (
type downloaderImpl struct {
progress aptly.Progress
aggWriter io.Writer
maxTries int
client *http.Client
}
// NewDownloader creates new instance of Downloader which specified number
// of threads and download limit in bytes/sec
func NewDownloader(downLimit int64, progress aptly.Progress) aptly.Downloader {
func NewDownloader(downLimit int64, maxTries int, progress aptly.Progress) aptly.Downloader {
transport := http.Transport{}
transport.Proxy = http.DefaultTransport.(*http.Transport).Proxy
transport.ResponseHeaderTimeout = 30 * time.Second
@@ -45,6 +47,7 @@ func NewDownloader(downLimit int64, progress aptly.Progress) aptly.Downloader {
downloader := &downloaderImpl{
progress: progress,
maxTries: maxTries,
client: &http.Client{
Transport: &transport,
},
@@ -71,7 +74,19 @@ func (downloader *downloaderImpl) GetLength(ctx context.Context, url string) (in
return -1, err
}
resp, err := downloader.client.Do(req)
var resp *http.Response
maxTries := downloader.maxTries
for maxTries > 0 {
resp, err = downloader.client.Do(req)
if err != nil && retryableError(err) {
maxTries--
} else {
// stop retrying
break
}
}
if err != nil {
return -1, errors.Wrap(err, url)
}
@@ -89,10 +104,25 @@ func (downloader *downloaderImpl) GetLength(ctx context.Context, url string) (in
// Download starts new download task
func (downloader *downloaderImpl) Download(ctx context.Context, url string, destination string) error {
return downloader.DownloadWithChecksum(ctx, url, destination, nil, false, 1)
return downloader.DownloadWithChecksum(ctx, url, destination, nil, false)
}
func retryableError(err error) bool {
// unwrap errors.Wrap
err = errors.Cause(err)
// unwrap *url.Error
if wrapped, ok := err.(*url.Error); ok {
err = wrapped.Err
}
switch err {
case io.EOF:
return true
case io.ErrUnexpectedEOF:
return true
}
switch err.(type) {
case *net.OpError:
return true
@@ -101,6 +131,7 @@ func retryableError(err error) bool {
case net.Error:
return true
}
return false
}
@@ -123,7 +154,7 @@ func (downloader *downloaderImpl) newRequest(ctx context.Context, method, url st
// DownloadWithChecksum starts new download task with checksum verification
func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url string, destination string,
expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
expected *utils.ChecksumInfo, ignoreMismatch bool) error {
if downloader.progress != nil {
downloader.progress.Printf("Downloading %s...\n", url)
@@ -131,6 +162,7 @@ func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url
req, err := downloader.newRequest(ctx, "GET", url)
var temppath string
maxTries := downloader.maxTries
for maxTries > 0 {
temppath, err = downloader.download(req, url, destination, expected, ignoreMismatch)