ignoreMismatch flag for downloading.

utils/download.go

@@ -9,12 +9,13 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 // Downloader is parallel HTTP fetcher
 type Downloader interface {
 	Download(url string, destination string, result chan<- error)
-	DownloadWithChecksum(url string, destination string, result chan<- error, expected ChecksumInfo)
+	DownloadWithChecksum(url string, destination string, result chan<- error, expected ChecksumInfo, ignoreMismatch bool)
 	Pause()
 	Resume()
 	Shutdown()
@@ -37,10 +38,11 @@ type downloaderImpl struct {
 
 // downloadTask represents single item in queue
 type downloadTask struct {
-	url         string
+	url            string
-	destination string
+	destination    string
-	result      chan<- error
+	result         chan<- error
-	expected    ChecksumInfo
+	expected       ChecksumInfo
+	ignoreMismatch bool
 }
 
 // NewDownloader creates new instance of Downloader which specified number
@@ -90,12 +92,13 @@ func (downloader *downloaderImpl) Resume() {
 
 // Download starts new download task
 func (downloader *downloaderImpl) Download(url string, destination string, result chan<- error) {
-	downloader.DownloadWithChecksum(url, destination, result, ChecksumInfo{Size: -1})
+	downloader.DownloadWithChecksum(url, destination, result, ChecksumInfo{Size: -1}, false)
 }
 
 // DownloadWithChecksum starts new download task with checksum verification
-func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, result chan<- error, expected ChecksumInfo) {
+func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, result chan<- error,
-	downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected}
+	expected ChecksumInfo, ignoreMismatch bool) {
+	downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch}
 }
 
 // handleTask processes single download task
@@ -160,9 +163,13 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 		}
 
 		if err != nil {
-			os.Remove(temppath)
+			if task.ignoreMismatch {
-			task.result <- err
+				fmt.Printf("WARNING: %s\n", err.Error())
-			return
+			} else {
+				os.Remove(temppath)
+				task.result <- err
+				return
+			}
 		}
 	}
 
@@ -195,6 +202,13 @@ func (downloader *downloaderImpl) process() {
 //
 // Temporary file would be already removed, so no need to cleanup
 func DownloadTemp(downloader Downloader, url string) (*os.File, error) {
+	return DownloadTempWithChecksum(downloader, url, ChecksumInfo{Size: -1}, false)
+}
+
+// DownloadTempWithChecksum is a DownloadTemp with checksum verification
+//
+// Temporary file would be already removed, so no need to cleanup
+func DownloadTempWithChecksum(downloader Downloader, url string, expected ChecksumInfo, ignoreMismatch bool) (*os.File, error) {
 	tempdir, err := ioutil.TempDir(os.TempDir(), "aptly")
 	if err != nil {
 		return nil, err
@@ -204,7 +218,7 @@ func DownloadTemp(downloader Downloader, url string) (*os.File, error) {
 	tempfile := filepath.Join(tempdir, "buffer")
 
 	ch := make(chan error, 1)
-	downloader.Download(url, tempfile, ch)
+	downloader.DownloadWithChecksum(url, tempfile, ch, expected, ignoreMismatch)
 
 	err = <-ch
 	if err != nil {
@@ -240,13 +254,27 @@ var compressionMethods = []struct {
 
 // DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until
 // it finds existing file.
-func DownloadTryCompression(downloader Downloader, url string) (io.Reader, *os.File, error) {
+func DownloadTryCompression(downloader Downloader, url string, expectedChecksums map[string]ChecksumInfo, ignoreMismatch bool) (io.Reader, *os.File, error) {
 	var err error
 
 	for _, method := range compressionMethods {
 		var file *os.File
 
-		file, err = DownloadTemp(downloader, url+method.extenstion)
+		tryUrl := url + method.extenstion
+		foundChecksum := false
+
+		for suffix, expected := range expectedChecksums {
+			if strings.HasSuffix(tryUrl, suffix) {
+				file, err = DownloadTempWithChecksum(downloader, tryUrl, expected, ignoreMismatch)
+				foundChecksum = true
+				break
+			}
+		}
+
+		if !foundChecksum {
+			file, err = DownloadTemp(downloader, tryUrl)
+		}
+
 		if err != nil {
 			continue
 		}

utils/download_test.go

@@ -91,34 +91,38 @@ func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
 	defer d.Shutdown()
 	ch := make(chan error)
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{})
+	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{}, false)
 	res := <-ch
 	c.Assert(res, ErrorMatches, ".*size check mismatch 12 != 0")
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "abcdef"})
+	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "abcdef"}, false)
 	res = <-ch
 	c.Assert(res, ErrorMatches, ".*md5 hash mismatch \"a1acb0fe91c7db45ec4d775192ec5738\" != \"abcdef\"")
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"})
+	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "abcdef"}, true)
 	res = <-ch
 	c.Assert(res, IsNil)
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"})
+	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}, false)
+	res = <-ch
+	c.Assert(res, IsNil)
+
+	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"}, false)
 	res = <-ch
 	c.Assert(res, ErrorMatches, ".*sha1 hash mismatch \"921893bae6ad6fd818401875d6779254ef0ff0ec\" != \"abcdef\"")
 
 	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"})
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"}, false)
 	res = <-ch
 	c.Assert(res, IsNil)
 
 	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"})
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"}, false)
 	res = <-ch
 	c.Assert(res, ErrorMatches, ".*sha256 hash mismatch \"b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac\" != \"abcdef\"")
 
 	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"})
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false)
 	res = <-ch
 	c.Assert(res, IsNil)
 }
@@ -170,6 +174,20 @@ func (s *DownloaderSuite) TestDownloadTemp(c *C) {
 	c.Assert(os.IsNotExist(err), Equals, true)
 }
 
+func (s *DownloaderSuite) TestDownloadTempWithChecksum(c *C) {
+	d := NewDownloader(2)
+	defer d.Shutdown()
+
+	f, err := DownloadTempWithChecksum(d, s.url+"/test", ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false)
+	defer f.Close()
+	c.Assert(err, IsNil)
+
+	f2, err := DownloadTempWithChecksum(d, s.url+"/test", ChecksumInfo{Size: 13}, false)
+	defer f2.Close()
+	c.Assert(err, ErrorMatches, ".*size check mismatch 12 != 13")
+}
+
 func (s *DownloaderSuite) TestDownloadTempError(c *C) {
 	d := NewDownloader(2)
 	defer d.Shutdown()
@@ -189,11 +207,17 @@ const (
 func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
 	var buf []byte
 
+	expectedChecksums := map[string]ChecksumInfo{
+		"file.bz2": ChecksumInfo{Size: int64(len(bzipData))},
+		"file.gz":  ChecksumInfo{Size: int64(len(gzipData))},
+		"file":     ChecksumInfo{Size: int64(len(rawData))},
+	}
+
 	// bzip2 only available
 	buf = make([]byte, 4)
 	d := NewFakeDownloader()
 	d.ExpectResponse("http://example.com/file.bz2", bzipData)
-	r, file, err := DownloadTryCompression(d, "http://example.com/file")
+	r, file, err := DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -205,7 +229,7 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
 	d.ExpectResponse("http://example.com/file.gz", gzipData)
-	r, file, err = DownloadTryCompression(d, "http://example.com/file")
+	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -218,7 +242,7 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
 	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
 	d.ExpectError("http://example.com/file.gz", errors.New("404"))
 	d.ExpectResponse("http://example.com/file", rawData)
-	r, file, err = DownloadTryCompression(d, "http://example.com/file")
+	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -231,7 +255,7 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
 	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
 	d.ExpectResponse("http://example.com/file.gz", "x")
 	d.ExpectResponse("http://example.com/file", "recovered")
-	r, file, err = DownloadTryCompression(d, "http://example.com/file")
+	r, file, err = DownloadTryCompression(d, "http://example.com/file", nil, false)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -241,13 +265,20 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
 
 func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) {
 	d := NewFakeDownloader()
-	_, _, err := DownloadTryCompression(d, "http://example.com/file")
+	_, _, err := DownloadTryCompression(d, "http://example.com/file", nil, false)
 	c.Assert(err, ErrorMatches, "unexpected request.*")
 
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
 	d.ExpectError("http://example.com/file.gz", errors.New("404"))
 	d.ExpectError("http://example.com/file", errors.New("403"))
-	_, _, err = DownloadTryCompression(d, "http://example.com/file")
+	_, _, err = DownloadTryCompression(d, "http://example.com/file", nil, false)
 	c.Assert(err, ErrorMatches, "403")
+
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
+	d.ExpectError("http://example.com/file.gz", errors.New("404"))
+	d.ExpectResponse("http://example.com/file", rawData)
+	_, _, err = DownloadTryCompression(d, "http://example.com/file", map[string]ChecksumInfo{"file": ChecksumInfo{Size: 7}}, false)
+	c.Assert(err, ErrorMatches, "checksums don't match.*")
 }

utils/fake.go

@@ -49,7 +49,7 @@ func (f *FakeDownloader) Empty() bool {
 }
 
 // DownloadWithChecksum performs fake download by matching against first expectation in the queue, with cheksum verification
-func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, result chan<- error, expected ChecksumInfo) {
+func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, result chan<- error, expected ChecksumInfo, ignoreMismatch bool) {
 	if len(f.expected) == 0 || f.expected[0].URL != url {
 		result <- fmt.Errorf("unexpected request for %s", url)
 		return
@@ -85,10 +85,15 @@ func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, resul
 		return
 	}
 
-	if expected.MD5 != "" {
+	if expected.Size != -1 {
-		if expected != cks.Sum() {
+		if expected.Size != cks.Sum().Size || expected.MD5 != "" && expected.MD5 != cks.Sum().MD5 ||
-			result <- fmt.Errorf("checksums don't match: %#v != %#v", expected, cks.Sum())
+			expected.SHA1 != "" && expected.SHA1 != cks.Sum().SHA1 || expected.SHA256 != "" && expected.SHA256 != cks.Sum().SHA256 {
-			return
+			if ignoreMismatch {
+				fmt.Printf("WARNING: checksums don't match: %#v != %#v\n", expected, cks.Sum())
+			} else {
+				result <- fmt.Errorf("checksums don't match: %#v != %#v", expected, cks.Sum())
+				return
+			}
 		}
 	}
 
@@ -98,7 +103,7 @@ func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, resul
 
 // Download performs fake download by matching against first expectation in the queue
 func (f *FakeDownloader) Download(url string, filename string, result chan<- error) {
-	f.DownloadWithChecksum(url, filename, result, ChecksumInfo{})
+	f.DownloadWithChecksum(url, filename, result, ChecksumInfo{Size: -1}, false)
 }
 
 // Shutdown does nothing