Add support to mirror non package installer files

http/compression.go

@@ -4,7 +4,6 @@ import (
 	"compress/bzip2"
 	"compress/gzip"
 	"context"
-	"fmt"
 	"io"
 	"net/url"
 	"os"
@@ -90,7 +89,7 @@ func DownloadTryCompression(ctx context.Context, downloader aptly.Downloader, ba
 	}
 
 	if err == nil {
-		err = fmt.Errorf("no candidates for %s found", baseURL.ResolveReference(&url.URL{Path: path}))
+		return nil, nil, &NoCandidateFoundError{URL: baseURL.ResolveReference(&url.URL{Path: path})}
 	}
 	return nil, nil, err
 }

http/download.go

@@ -64,6 +64,29 @@ func (downloader *downloaderImpl) GetProgress() aptly.Progress {
 	return downloader.progress
 }
 
+// GetLength of given url
+func (downloader *downloaderImpl) GetLength(ctx context.Context, url string) (int64, error) {
+	req, err := downloader.newRequest(ctx, "HEAD", url)
+	if err != nil {
+		return -1, err
+	}
+
+	resp, err := downloader.client.Do(req)
+	if err != nil {
+		return -1, errors.Wrap(err, url)
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return -1, &Error{Code: resp.StatusCode, URL: url}
+	}
+
+	if resp.ContentLength < 0 {
+		return -1, fmt.Errorf("Could not determine length of %s", url)
+	}
+
+	return resp.ContentLength, nil
+}
+
 // Download starts new download task
 func (downloader *downloaderImpl) Download(ctx context.Context, url string, destination string) error {
 	return downloader.DownloadWithChecksum(ctx, url, destination, nil, false, 1)
@@ -81,15 +104,10 @@ func retryableError(err error) bool {
 	return false
 }
 
-// DownloadWithChecksum starts new download task with checksum verification
-func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url string, destination string,
-	expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
-
-	downloader.progress.Printf("Downloading %s...\n", url)
-
-	req, err := http.NewRequest("GET", url, nil)
+func (downloader *downloaderImpl) newRequest(ctx context.Context, method, url string) (*http.Request, error) {
+	req, err := http.NewRequest(method, url, nil)
 	if err != nil {
-		return errors.Wrap(err, url)
+		return nil, errors.Wrap(err, url)
 	}
 	req.Close = true
 	req = req.WithContext(ctx)
@@ -100,6 +118,18 @@ func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url
 		req.URL.RawQuery = ""
 	}
 
+	return req, nil
+}
+
+// DownloadWithChecksum starts new download task with checksum verification
+func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url string, destination string,
+	expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
+
+	if downloader.progress != nil {
+		downloader.progress.Printf("Downloading %s...\n", url)
+	}
+	req, err := downloader.newRequest(ctx, "GET", url)
+
 	var temppath string
 	for maxTries > 0 {
 		temppath, err = downloader.download(req, url, destination, expected, ignoreMismatch)

http/fake.go

@@ -60,8 +60,17 @@ func (f *FakeDownloader) Empty() bool {
 	return len(f.expected) == 0
 }
 
-// DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification
-func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
+// GetLength returns content length of given url
+func (f *FakeDownloader) GetLength(ctx context.Context, url string) (int64, error) {
+	expectation, err := f.getExpectedRequest(url)
+	if err != nil {
+		return -1, err
+	}
+
+	return int64(len(expectation.Response)), nil
+}
+
+func (f *FakeDownloader) getExpectedRequest(url string) (*expectedRequest, error) {
 	var expectation expectedRequest
 	if len(f.expected) > 0 && f.expected[0].URL == url {
 		expectation, f.expected = f.expected[0], f.expected[1:]
@@ -69,14 +78,24 @@ func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, f
 		expectation = f.anyExpected[url]
 		delete(f.anyExpected, url)
 	} else {
-		return fmt.Errorf("unexpected request for %s", url)
+		return nil, fmt.Errorf("unexpected request for %s", url)
 	}
 
 	if expectation.Err != nil {
-		return expectation.Err
+		return nil, expectation.Err
 	}
 
-	err := os.MkdirAll(filepath.Dir(filename), 0755)
+	return &expectation, nil
+}
+
+// DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification
+func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
+	expectation, err := f.getExpectedRequest(url)
+	if err != nil {
+		return err
+	}
+
+	err = os.MkdirAll(filepath.Dir(filename), 0755)
 	if err != nil {
 		return err
 	}

http/http.go

@@ -3,6 +3,7 @@ package http
 
 import (
 	"fmt"
+	"net/url"
 )
 
 // Error is download error connected to HTTP code
@@ -15,3 +16,13 @@ type Error struct {
 func (e *Error) Error() string {
 	return fmt.Sprintf("HTTP code %d while fetching %s", e.Code, e.URL)
 }
+
+// NoCandidateFoundError indicates that now candidate of given url could be found
+type NoCandidateFoundError struct {
+	URL *url.URL
+}
+
+// Error message
+func (e *NoCandidateFoundError) Error() string {
+	return fmt.Sprintf("no candidates for %s found", e.URL)
+}