New upstream version 1.5.0+ds1

http/compression.go

@@ -39,7 +39,7 @@ var compressionMethods = []struct {
 
 // DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until
 // it finds existing file.
-func DownloadTryCompression(ctx context.Context, downloader aptly.Downloader, baseURL *url.URL, path string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (io.Reader, *os.File, error) {
+func DownloadTryCompression(ctx context.Context, downloader aptly.Downloader, baseURL *url.URL, path string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool) (io.Reader, *os.File, error) {
 	var err error
 
 	for _, method := range compressionMethods {
@@ -63,7 +63,7 @@ func DownloadTryCompression(ctx context.Context, downloader aptly.Downloader, ba
 
 		if foundChecksum {
 			expected := expectedChecksums[bestSuffix]
-			file, err = DownloadTempWithChecksum(ctx, downloader, tryURL.String(), &expected, ignoreMismatch, maxTries)
+			file, err = DownloadTempWithChecksum(ctx, downloader, tryURL.String(), &expected, ignoreMismatch)
 		} else {
 			if !ignoreMismatch {
 				continue

http/compression_test.go

@@ -44,7 +44,7 @@ func (s *CompressionSuite) TestDownloadTryCompression(c *C) {
 	buf = make([]byte, 4)
 	d := NewFakeDownloader()
 	d.ExpectResponse("http://example.com/file.bz2", bzipData)
-	r, file, err := DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
+	r, file, err := DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -56,7 +56,7 @@ func (s *CompressionSuite) TestDownloadTryCompression(c *C) {
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
 	d.ExpectResponse("http://example.com/file.gz", gzipData)
-	r, file, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
+	r, file, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -69,7 +69,7 @@ func (s *CompressionSuite) TestDownloadTryCompression(c *C) {
 	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
 	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
 	d.ExpectResponse("http://example.com/file.xz", xzData)
-	r, file, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
+	r, file, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -83,7 +83,7 @@ func (s *CompressionSuite) TestDownloadTryCompression(c *C) {
 	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
 	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
 	d.ExpectResponse("http://example.com/file", rawData)
-	r, file, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
+	r, file, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -94,7 +94,7 @@ func (s *CompressionSuite) TestDownloadTryCompression(c *C) {
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
 	d.ExpectResponse("http://example.com/file.gz", "x")
-	_, _, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", nil, true, 1)
+	_, _, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", nil, true)
 	c.Assert(err, ErrorMatches, "unexpected EOF")
 	c.Assert(d.Empty(), Equals, true)
 }
@@ -112,7 +112,7 @@ func (s *CompressionSuite) TestDownloadTryCompressionLongestSuffix(c *C) {
 	buf = make([]byte, 4)
 	d := NewFakeDownloader()
 	d.ExpectResponse("http://example.com/subdir/file.bz2", bzipData)
-	r, file, err := DownloadTryCompression(s.ctx, d, s.baseURL, "subdir/file", expectedChecksums, false, 1)
+	r, file, err := DownloadTryCompression(s.ctx, d, s.baseURL, "subdir/file", expectedChecksums, false)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -122,7 +122,7 @@ func (s *CompressionSuite) TestDownloadTryCompressionLongestSuffix(c *C) {
 
 func (s *CompressionSuite) TestDownloadTryCompressionErrors(c *C) {
 	d := NewFakeDownloader()
-	_, _, err := DownloadTryCompression(s.ctx, d, s.baseURL, "file", nil, true, 1)
+	_, _, err := DownloadTryCompression(s.ctx, d, s.baseURL, "file", nil, true)
 	c.Assert(err, ErrorMatches, "unexpected request.*")
 
 	d = NewFakeDownloader()
@@ -130,7 +130,7 @@ func (s *CompressionSuite) TestDownloadTryCompressionErrors(c *C) {
 	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
 	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
 	d.ExpectError("http://example.com/file", errors.New("403"))
-	_, _, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", nil, true, 1)
+	_, _, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", nil, true)
 	c.Assert(err, ErrorMatches, "403")
 
 	d = NewFakeDownloader()
@@ -144,6 +144,6 @@ func (s *CompressionSuite) TestDownloadTryCompressionErrors(c *C) {
 		"file.xz":  {Size: 7},
 		"file":     {Size: 7},
 	}
-	_, _, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
+	_, _, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false)
 	c.Assert(err, ErrorMatches, "checksums don't match.*")
 }

http/download.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -28,12 +30,13 @@ var (
 type downloaderImpl struct {
 	progress  aptly.Progress
 	aggWriter io.Writer
+	maxTries  int
 	client    *http.Client
 }
 
 // NewDownloader creates new instance of Downloader which specified number
 // of threads and download limit in bytes/sec
-func NewDownloader(downLimit int64, progress aptly.Progress) aptly.Downloader {
+func NewDownloader(downLimit int64, maxTries int, progress aptly.Progress) aptly.Downloader {
 	transport := http.Transport{}
 	transport.Proxy = http.DefaultTransport.(*http.Transport).Proxy
 	transport.ResponseHeaderTimeout = 30 * time.Second
@@ -45,20 +48,35 @@ func NewDownloader(downLimit int64, progress aptly.Progress) aptly.Downloader {
 
 	downloader := &downloaderImpl{
 		progress: progress,
+		maxTries: maxTries,
 		client: &http.Client{
 			Transport: &transport,
 		},
 	}
 
+	progressWriter := io.Writer(progress)
+	if progress == nil {
+		progressWriter = ioutil.Discard
+	}
+
+	downloader.client.CheckRedirect = downloader.checkRedirect
 	if downLimit > 0 {
-		downloader.aggWriter = flowrate.NewWriter(progress, downLimit)
+		downloader.aggWriter = flowrate.NewWriter(progressWriter, downLimit)
 	} else {
-		downloader.aggWriter = progress
+		downloader.aggWriter = progressWriter
 	}
 
 	return downloader
 }
 
+func (downloader *downloaderImpl) checkRedirect(req *http.Request, via []*http.Request) error {
+	if downloader.progress != nil {
+		downloader.progress.Printf("Following redirect to %s...\n", req.URL)
+	}
+
+	return nil
+}
+
 // GetProgress returns Progress object
 func (downloader *downloaderImpl) GetProgress() aptly.Progress {
 	return downloader.progress
@@ -71,7 +89,19 @@ func (downloader *downloaderImpl) GetLength(ctx context.Context, url string) (in
 		return -1, err
 	}
 
-	resp, err := downloader.client.Do(req)
+	var resp *http.Response
+
+	maxTries := downloader.maxTries
+	for maxTries > 0 {
+		resp, err = downloader.client.Do(req)
+		if err != nil && retryableError(err) {
+			maxTries--
+		} else {
+			// stop retrying
+			break
+		}
+	}
+
 	if err != nil {
 		return -1, errors.Wrap(err, url)
 	}
@@ -81,6 +111,10 @@ func (downloader *downloaderImpl) GetLength(ctx context.Context, url string) (in
 	}
 
 	if resp.ContentLength < 0 {
+		// an existing, but zero-length file can be reported with ContentLength -1
+		if resp.StatusCode == 200 && resp.ContentLength == -1 {
+			return 0, nil
+		}
 		return -1, fmt.Errorf("could not determine length of %s", url)
 	}
 
@@ -89,10 +123,25 @@ func (downloader *downloaderImpl) GetLength(ctx context.Context, url string) (in
 
 // Download starts new download task
 func (downloader *downloaderImpl) Download(ctx context.Context, url string, destination string) error {
-	return downloader.DownloadWithChecksum(ctx, url, destination, nil, false, 1)
+	return downloader.DownloadWithChecksum(ctx, url, destination, nil, false)
 }
 
 func retryableError(err error) bool {
+	// unwrap errors.Wrap
+	err = errors.Cause(err)
+
+	// unwrap *url.Error
+	if wrapped, ok := err.(*url.Error); ok {
+		err = wrapped.Err
+	}
+
+	switch err {
+	case io.EOF:
+		return true
+	case io.ErrUnexpectedEOF:
+		return true
+	}
+
 	switch err.(type) {
 	case *net.OpError:
 		return true
@@ -101,11 +150,13 @@ func retryableError(err error) bool {
 	case net.Error:
 		return true
 	}
-	return false
+	// Note: make all errors retryable
+	return true
 }
 
 func (downloader *downloaderImpl) newRequest(ctx context.Context, method, url string) (*http.Request, error) {
 	req, err := http.NewRequest(method, url, nil)
+
 	if err != nil {
 		return nil, errors.Wrap(err, url)
 	}
@@ -123,27 +174,57 @@ func (downloader *downloaderImpl) newRequest(ctx context.Context, method, url st
 
 // DownloadWithChecksum starts new download task with checksum verification
 func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url string, destination string,
-	expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
+	expected *utils.ChecksumInfo, ignoreMismatch bool) error {
 
 	if downloader.progress != nil {
 		downloader.progress.Printf("Downloading %s...\n", url)
+		defer downloader.progress.Flush()
 	}
 	req, err := downloader.newRequest(ctx, "GET", url)
 
 	var temppath string
+	maxTries := downloader.maxTries
+	const delayMax = time.Duration(5 * time.Minute)
+	delay := time.Duration(1 * time.Second)
+	const delayMultiplier = 2
 	for maxTries > 0 {
 		temppath, err = downloader.download(req, url, destination, expected, ignoreMismatch)
 
-		if err != nil && retryableError(err) {
-			maxTries--
+		if err != nil {
+			if retryableError(err) {
+				if downloader.progress != nil {
+					downloader.progress.Printf("Error downloading %s: %s retrying...\n", url, err)
+				}
+				maxTries--
+				time.Sleep(delay)
+				// Sleep exponentially at the next retry, but no longer than delayMax
+				delay *= delayMultiplier
+				if delay > delayMax {
+					delay = delayMax
+				}
+			} else {
+				if downloader.progress != nil {
+					downloader.progress.Printf("Error downloading %s: %s cannot retry...\n", url, err)
+				}
+				break
+			}
 		} else {
 			// get out of the loop
+			if downloader.progress != nil {
+				downloader.progress.Printf("Success downloading %s\n", url)
+			}
 			break
 		}
+		if downloader.progress != nil {
+			downloader.progress.Printf("Retrying %d %s...\n", maxTries, url)
+		}
 	}
 
 	// still an error after retrying, giving up
 	if err != nil {
+		if downloader.progress != nil {
+			downloader.progress.Printf("Giving up on %s...\n", url)
+		}
 		return err
 	}
 
@@ -183,7 +264,11 @@ func (downloader *downloaderImpl) download(req *http.Request, url, destination s
 	defer outfile.Close()
 
 	checksummer := utils.NewChecksumWriter()
-	writers := []io.Writer{outfile, downloader.aggWriter}
+	writers := []io.Writer{outfile}
+
+	if downloader.progress != nil {
+		writers = append(writers, downloader.progress)
+	}
 
 	if expected != nil {
 		writers = append(writers, checksummer)
@@ -214,7 +299,9 @@ func (downloader *downloaderImpl) download(req *http.Request, url, destination s
 
 		if err != nil {
 			if ignoreMismatch {
-				downloader.progress.Printf("WARNING: %s\n", err.Error())
+				if downloader.progress != nil {
+					downloader.progress.Printf("WARNING: %s\n", err.Error())
+				}
 			} else {
 				os.Remove(temppath)
 				return "", err

http/download_go17.go

@@ -1,3 +1,4 @@
+//go:build go1.7
 // +build go1.7
 
 package http

http/download_test.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"os/user"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/console"
@@ -45,7 +46,7 @@ func (s *DownloaderSuiteBase) SetUpTest(c *C) {
 	s.progress = console.NewProgress()
 	s.progress.Start()
 
-	s.d = NewDownloader(0, s.progress)
+	s.d = NewDownloader(0, 1, s.progress)
 	s.ctx = context.Background()
 }
 
@@ -78,32 +79,32 @@ func (s *DownloaderSuite) TestDownloadOK(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
-	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{}, false, 1),
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{}, false),
 		ErrorMatches, ".*size check mismatch 12 != 0")
 
-	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, false, 1),
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, false),
 		ErrorMatches, ".*md5 hash mismatch \"a1acb0fe91c7db45ec4d775192ec5738\" != \"abcdef\"")
 
-	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, true, 1),
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, true),
 		IsNil)
 
-	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}, false, 1),
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}, false),
 		IsNil)
 
-	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"}, false, 1),
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"}, false),
 		ErrorMatches, ".*sha1 hash mismatch \"921893bae6ad6fd818401875d6779254ef0ff0ec\" != \"abcdef\"")
 
 	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"}, false, 1),
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"}, false),
 		IsNil)
 
 	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"}, false, 1),
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"}, false),
 		ErrorMatches, ".*sha256 hash mismatch \"b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac\" != \"abcdef\"")
 
 	checksums := utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
 		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}
-	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &checksums, false, 1),
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &checksums, false),
 		IsNil)
 	// download backfills missing checksums
 	c.Check(checksums.SHA512, Equals, "bac18bf4e564856369acc2ed57300fecba3a2c1af5ae8304021e4252488678feb18118466382ee4e1210fe1f065080210e453a80cfb37ccb8752af3269df160e")
@@ -115,13 +116,25 @@ func (s *DownloaderSuite) TestDownload404(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadConnectError(c *C) {
-	c.Assert(s.d.Download(s.ctx, "http://nosuch.localhost/", s.tempfile.Name()),
+	c.Assert(s.d.Download(s.ctx, "http://nosuch.host/", s.tempfile.Name()),
 		ErrorMatches, ".*no such host")
 }
 
+func skipIfRoot(c *C) {
+	currentUser, err := user.Current()
+	if err != nil {
+		c.Skip("Unknown user")
+	}
+
+	if currentUser.Username == "root" {
+		c.Skip("Root user")
+	}
+}
+
 func (s *DownloaderSuite) TestDownloadFileError(c *C) {
+	skipIfRoot(c)
 	c.Assert(s.d.Download(s.ctx, s.url+"/test", "/"),
-		ErrorMatches, ".*permission denied")
+		ErrorMatches, ".*(permission denied|read-only file system)")
 }
 
 func (s *DownloaderSuite) TestGetLength(c *C) {
@@ -138,7 +151,7 @@ func (s *DownloaderSuite) TestGetLength404(c *C) {
 }
 
 func (s *DownloaderSuite) TestGetLengthConnectError(c *C) {
-	_, err := s.d.GetLength(s.ctx, "http://nosuch.localhost/")
+	_, err := s.d.GetLength(s.ctx, "http://nosuch.host/")
 
 	c.Assert(err, ErrorMatches, ".*no such host")
 }

http/fake.go

@@ -89,7 +89,7 @@ func (f *FakeDownloader) getExpectedRequest(url string) (*expectedRequest, error
 }
 
 // DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification
-func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
+func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool) error {
 	expectation, err := f.getExpectedRequest(url)
 	if err != nil {
 		return err
@@ -130,7 +130,7 @@ func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, f
 
 // Download performs fake download by matching against first expectation in the queue
 func (f *FakeDownloader) Download(ctx context.Context, url string, filename string) error {
-	return f.DownloadWithChecksum(ctx, url, filename, nil, false, 1)
+	return f.DownloadWithChecksum(ctx, url, filename, nil, false)
 }
 
 // GetProgress returns Progress object

http/grab.go

@@ -0,0 +1,173 @@
+package http
+
+import (
+	"context"
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/hex"
+	"fmt"
+	"net/http"
+	"time"
+
+	"golang.org/x/time/rate"
+
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/cavaliergopher/grab/v3"
+	"github.com/pkg/errors"
+
+	"github.com/aptly-dev/aptly/aptly"
+)
+
+type GrabDownloader struct {
+	client    *grab.Client
+	progress  aptly.Progress
+	maxTries  int
+	downLimit int64
+}
+
+// Check interface
+var (
+	_ aptly.Downloader = (*GrabDownloader)(nil)
+)
+
+// NewGrabDownloader creates new expected downloader
+func NewGrabDownloader(downLimit int64, maxTries int, progress aptly.Progress) *GrabDownloader {
+	client := grab.NewClient()
+	return &GrabDownloader{
+		client:    client,
+		progress:  progress,
+		maxTries:  maxTries,
+		downLimit: downLimit,
+	}
+}
+
+func (d *GrabDownloader) Download(ctx context.Context, url string, destination string) error {
+	return d.DownloadWithChecksum(ctx, url, destination, nil, false)
+}
+
+func (d *GrabDownloader) DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error {
+	maxTries := d.maxTries
+	const delayMax = time.Duration(5 * time.Minute)
+	delay := time.Duration(1 * time.Second)
+	const delayMultiplier = 2
+	err := fmt.Errorf("No tries available")
+	for maxTries > 0 {
+		err = d.download(ctx, url, destination, expected, ignoreMismatch)
+		if err == nil {
+			// Success
+			break
+		}
+		d.log("Error downloading %s: %v\n", url, err)
+		if retryableError(err) {
+			maxTries--
+			d.log("Retrying download %s: %d\n", url, maxTries)
+			time.Sleep(delay)
+		} else {
+			// Can't retry
+			d.log("Cannot retry download %s\n", url)
+			break
+		}
+	}
+	return err
+}
+
+func (d *GrabDownloader) log(msg string, a ...interface{}) {
+	fmt.Printf(msg, a...)
+	if d.progress != nil {
+		d.progress.Printf(msg, a...)
+	}
+}
+
+func (d *GrabDownloader) maybeSetupChecksum(req *grab.Request, expected *utils.ChecksumInfo) error {
+	if expected == nil {
+		// Nothing to setup
+		return nil
+	}
+	if expected.MD5 != "" {
+		expectedHash, err := hex.DecodeString(expected.MD5)
+		if err != nil {
+			return err
+		}
+		req.SetChecksum(md5.New(), expectedHash, true)
+	} else if expected.SHA1 != "" {
+		expectedHash, err := hex.DecodeString(expected.SHA1)
+		if err != nil {
+			return err
+		}
+		req.SetChecksum(sha1.New(), expectedHash, true)
+	} else if expected.SHA256 != "" {
+		expectedHash, err := hex.DecodeString(expected.SHA256)
+		if err != nil {
+			return err
+		}
+		req.SetChecksum(sha256.New(), expectedHash, true)
+	} else if expected.SHA512 != "" {
+		expectedHash, err := hex.DecodeString(expected.SHA512)
+		if err != nil {
+			return err
+		}
+		req.SetChecksum(sha512.New(), expectedHash, true)
+	}
+	req.Size = expected.Size
+	return nil
+}
+
+func (d *GrabDownloader) download(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error {
+	// TODO clean up dest dir on permanent failure
+	d.log("Download %s -> %s\n", url, destination)
+
+	req, err := grab.NewRequest(destination, url)
+	if err != nil {
+		d.log("Error creating new request: %v\n", err)
+		return errors.Wrap(err, url)
+	}
+	if d.downLimit > 0 {
+		req.RateLimiter = rate.NewLimiter(rate.Limit(d.downLimit), int(d.downLimit))
+	}
+
+	d.maybeSetupChecksum(req, expected)
+	if err != nil {
+		d.log("Error setting up checksum: %v\n", err)
+		return errors.Wrap(err, url)
+	}
+
+	resp := d.client.Do(req)
+
+Loop:
+	for {
+		select {
+		case <-resp.Done:
+			// download is complete
+			break Loop
+		}
+	}
+	err = resp.Err()
+	if err != nil && err == grab.ErrBadChecksum && ignoreMismatch {
+		fmt.Printf("Ignoring checksum mismatch for %s\n", url)
+		return nil
+	}
+	return err
+}
+
+func (d *GrabDownloader) GetProgress() aptly.Progress {
+	return d.progress
+}
+
+func (d *GrabDownloader) GetLength(ctx context.Context, url string) (int64, error) {
+	resp, err := http.Head(url)
+	if err != nil {
+		return -1, err
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return -1, &Error{Code: resp.StatusCode, URL: url}
+	}
+
+	if resp.ContentLength < 0 {
+		return -1, fmt.Errorf("could not determine length of %s", url)
+	}
+
+	return resp.ContentLength, nil
+}

http/grab_test.go

@@ -0,0 +1,137 @@
+package http
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/console"
+	"github.com/aptly-dev/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type GrabDownloaderSuiteBase struct {
+	tempfile *os.File
+	l        net.Listener
+	url      string
+	ch       chan struct{}
+	progress aptly.Progress
+	d        aptly.Downloader
+	ctx      context.Context
+}
+
+func (s *GrabDownloaderSuiteBase) SetUpTest(c *C) {
+	s.tempfile, _ = ioutil.TempFile(os.TempDir(), "aptly-test")
+	s.l, _ = net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)})
+	s.url = fmt.Sprintf("http://localhost:%d", s.l.Addr().(*net.TCPAddr).Port)
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "Hello, %s", r.URL.Path)
+	})
+
+	s.ch = make(chan struct{})
+
+	go func() {
+		http.Serve(s.l, mux)
+		close(s.ch)
+	}()
+
+	s.progress = console.NewProgress()
+	s.progress.Start()
+
+	s.d = NewGrabDownloader(0, 1, s.progress)
+	s.ctx = context.Background()
+}
+
+func (s *GrabDownloaderSuiteBase) TearDownTest(c *C) {
+	s.progress.Shutdown()
+
+	s.l.Close()
+	<-s.ch
+
+	os.Remove(s.tempfile.Name())
+	s.tempfile.Close()
+}
+
+type GrabDownloaderSuite struct {
+	GrabDownloaderSuiteBase
+}
+
+var _ = Suite(&GrabDownloaderSuite{})
+
+func (s *GrabDownloaderSuite) SetUpTest(c *C) {
+	s.GrabDownloaderSuiteBase.SetUpTest(c)
+}
+
+func (s *GrabDownloaderSuite) TearDownTest(c *C) {
+	s.GrabDownloaderSuiteBase.TearDownTest(c)
+}
+
+func (s *GrabDownloaderSuite) TestDownloadOK(c *C) {
+	c.Assert(s.d.Download(s.ctx, s.url+"/test", s.tempfile.Name()), IsNil)
+}
+
+func (s *GrabDownloaderSuite) TestDownloadWithChecksum(c *C) {
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 1}, false),
+		// ErrorMatches, ".*size check mismatch 12 != 1")
+		ErrorMatches, "bad content length")
+
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, false),
+		ErrorMatches, "checksum mismatch")
+
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, true),
+		IsNil)
+
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}, false),
+		IsNil)
+
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"}, false),
+		IsNil)
+
+	checksums := utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &checksums, false),
+		IsNil)
+}
+
+func (s *GrabDownloaderSuite) TestDownload404(c *C) {
+	c.Assert(s.d.Download(s.ctx, s.url+"/doesntexist", s.tempfile.Name()),
+		ErrorMatches, ".* 404 .*")
+}
+
+func (s *GrabDownloaderSuite) TestDownloadConnectError(c *C) {
+	c.Assert(s.d.Download(s.ctx, "http://nosuch.host/", s.tempfile.Name()),
+		ErrorMatches, ".*no such host")
+}
+
+func (s *GrabDownloaderSuite) TestDownloadFileError(c *C) {
+	skipIfRoot(c)
+	c.Assert(s.d.Download(s.ctx, s.url+"/test", "/"),
+		ErrorMatches, ".*(permission denied|read-only file system)")
+}
+
+func (s *GrabDownloaderSuite) TestGetLength(c *C) {
+	size, err := s.d.GetLength(s.ctx, s.url+"/test")
+
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(12))
+}
+
+func (s *GrabDownloaderSuite) TestGetLength404(c *C) {
+	_, err := s.d.GetLength(s.ctx, s.url+"/doesntexist")
+
+	c.Assert(err, ErrorMatches, "HTTP code 404.*")
+}
+
+func (s *GrabDownloaderSuite) TestGetLengthConnectError(c *C) {
+	_, err := s.d.GetLength(s.ctx, "http://nosuch.host/")
+
+	c.Assert(err, ErrorMatches, ".*no such host")
+}

http/temp.go

@@ -14,13 +14,13 @@ import (
 //
 // Temporary file would be already removed, so no need to cleanup
 func DownloadTemp(ctx context.Context, downloader aptly.Downloader, url string) (*os.File, error) {
-	return DownloadTempWithChecksum(ctx, downloader, url, nil, false, 1)
+	return DownloadTempWithChecksum(ctx, downloader, url, nil, false)
 }
 
 // DownloadTempWithChecksum is a DownloadTemp with checksum verification
 //
 // Temporary file would be already removed, so no need to cleanup
-func DownloadTempWithChecksum(ctx context.Context, downloader aptly.Downloader, url string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (*os.File, error) {
+func DownloadTempWithChecksum(ctx context.Context, downloader aptly.Downloader, url string, expected *utils.ChecksumInfo, ignoreMismatch bool) (*os.File, error) {
 	tempdir, err := ioutil.TempDir(os.TempDir(), "aptly")
 	if err != nil {
 		return nil, err
@@ -30,11 +30,11 @@ func DownloadTempWithChecksum(ctx context.Context, downloader aptly.Downloader,
 	tempfile := filepath.Join(tempdir, "buffer")
 
 	if expected != nil && downloader.GetProgress() != nil {
-		downloader.GetProgress().InitBar(expected.Size, true)
+		downloader.GetProgress().InitBar(expected.Size, true, aptly.BarMirrorUpdateDownloadIndexes)
 		defer downloader.GetProgress().ShutdownBar()
 	}
 
-	err = downloader.DownloadWithChecksum(ctx, url, tempfile, expected, ignoreMismatch, maxTries)
+	err = downloader.DownloadWithChecksum(ctx, url, tempfile, expected, ignoreMismatch)
 	if err != nil {
 		return nil, err
 	}

http/temp_test.go

@@ -38,12 +38,12 @@ func (s *TempSuite) TestDownloadTemp(c *C) {
 
 func (s *TempSuite) TestDownloadTempWithChecksum(c *C) {
 	f, err := DownloadTempWithChecksum(s.ctx, s.d, s.url+"/test", &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false, 1)
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false)
 	c.Assert(err, IsNil)
 
 	c.Assert(f.Close(), IsNil)
 
-	_, err = DownloadTempWithChecksum(s.ctx, s.d, s.url+"/test", &utils.ChecksumInfo{Size: 13}, false, 1)
+	_, err = DownloadTempWithChecksum(s.ctx, s.d, s.url+"/test", &utils.ChecksumInfo{Size: 13}, false)
 	c.Assert(err, ErrorMatches, ".*size check mismatch 12 != 13")
 }