diff --git a/utils/download.go b/utils/download.go index bf7bd256..3ca06f11 100644 --- a/utils/download.go +++ b/utils/download.go @@ -9,12 +9,13 @@ import ( "net/http" "os" "path/filepath" + "strings" ) // Downloader is parallel HTTP fetcher type Downloader interface { Download(url string, destination string, result chan<- error) - DownloadWithChecksum(url string, destination string, result chan<- error, expected ChecksumInfo) + DownloadWithChecksum(url string, destination string, result chan<- error, expected ChecksumInfo, ignoreMismatch bool) Pause() Resume() Shutdown() @@ -37,10 +38,11 @@ type downloaderImpl struct { // downloadTask represents single item in queue type downloadTask struct { - url string - destination string - result chan<- error - expected ChecksumInfo + url string + destination string + result chan<- error + expected ChecksumInfo + ignoreMismatch bool } // NewDownloader creates new instance of Downloader which specified number @@ -90,12 +92,13 @@ func (downloader *downloaderImpl) Resume() { // Download starts new download task func (downloader *downloaderImpl) Download(url string, destination string, result chan<- error) { - downloader.DownloadWithChecksum(url, destination, result, ChecksumInfo{Size: -1}) + downloader.DownloadWithChecksum(url, destination, result, ChecksumInfo{Size: -1}, false) } // DownloadWithChecksum starts new download task with checksum verification -func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, result chan<- error, expected ChecksumInfo) { - downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected} +func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, result chan<- error, + expected ChecksumInfo, ignoreMismatch bool) { + downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch} } // handleTask processes single download task @@ -160,9 +163,13 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) { } if err != nil { - os.Remove(temppath) - task.result <- err - return + if task.ignoreMismatch { + fmt.Printf("WARNING: %s\n", err.Error()) + } else { + os.Remove(temppath) + task.result <- err + return + } } } @@ -195,6 +202,13 @@ func (downloader *downloaderImpl) process() { // // Temporary file would be already removed, so no need to cleanup func DownloadTemp(downloader Downloader, url string) (*os.File, error) { + return DownloadTempWithChecksum(downloader, url, ChecksumInfo{Size: -1}, false) +} + +// DownloadTempWithChecksum is a DownloadTemp with checksum verification +// +// Temporary file would be already removed, so no need to cleanup +func DownloadTempWithChecksum(downloader Downloader, url string, expected ChecksumInfo, ignoreMismatch bool) (*os.File, error) { tempdir, err := ioutil.TempDir(os.TempDir(), "aptly") if err != nil { return nil, err @@ -204,7 +218,7 @@ func DownloadTemp(downloader Downloader, url string) (*os.File, error) { tempfile := filepath.Join(tempdir, "buffer") ch := make(chan error, 1) - downloader.Download(url, tempfile, ch) + downloader.DownloadWithChecksum(url, tempfile, ch, expected, ignoreMismatch) err = <-ch if err != nil { @@ -240,13 +254,27 @@ var compressionMethods = []struct { // DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until // it finds existing file. -func DownloadTryCompression(downloader Downloader, url string) (io.Reader, *os.File, error) { +func DownloadTryCompression(downloader Downloader, url string, expectedChecksums map[string]ChecksumInfo, ignoreMismatch bool) (io.Reader, *os.File, error) { var err error for _, method := range compressionMethods { var file *os.File - file, err = DownloadTemp(downloader, url+method.extenstion) + tryUrl := url + method.extenstion + foundChecksum := false + + for suffix, expected := range expectedChecksums { + if strings.HasSuffix(tryUrl, suffix) { + file, err = DownloadTempWithChecksum(downloader, tryUrl, expected, ignoreMismatch) + foundChecksum = true + break + } + } + + if !foundChecksum { + file, err = DownloadTemp(downloader, tryUrl) + } + if err != nil { continue } diff --git a/utils/download_test.go b/utils/download_test.go index 0a2c6ab0..947b455b 100644 --- a/utils/download_test.go +++ b/utils/download_test.go @@ -91,34 +91,38 @@ func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) { defer d.Shutdown() ch := make(chan error) - d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{}) + d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{}, false) res := <-ch c.Assert(res, ErrorMatches, ".*size check mismatch 12 != 0") - d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "abcdef"}) + d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "abcdef"}, false) res = <-ch c.Assert(res, ErrorMatches, ".*md5 hash mismatch \"a1acb0fe91c7db45ec4d775192ec5738\" != \"abcdef\"") - d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}) + d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "abcdef"}, true) res = <-ch c.Assert(res, IsNil) - d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"}) + d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}, false) + res = <-ch + c.Assert(res, IsNil) + + d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"}, false) res = <-ch c.Assert(res, ErrorMatches, ".*sha1 hash mismatch \"921893bae6ad6fd818401875d6779254ef0ff0ec\" != \"abcdef\"") d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", - SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"}) + SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"}, false) res = <-ch c.Assert(res, IsNil) d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", - SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"}) + SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"}, false) res = <-ch c.Assert(res, ErrorMatches, ".*sha256 hash mismatch \"b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac\" != \"abcdef\"") d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", - SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}) + SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false) res = <-ch c.Assert(res, IsNil) } @@ -170,6 +174,20 @@ func (s *DownloaderSuite) TestDownloadTemp(c *C) { c.Assert(os.IsNotExist(err), Equals, true) } +func (s *DownloaderSuite) TestDownloadTempWithChecksum(c *C) { + d := NewDownloader(2) + defer d.Shutdown() + + f, err := DownloadTempWithChecksum(d, s.url+"/test", ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", + SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false) + defer f.Close() + c.Assert(err, IsNil) + + f2, err := DownloadTempWithChecksum(d, s.url+"/test", ChecksumInfo{Size: 13}, false) + defer f2.Close() + c.Assert(err, ErrorMatches, ".*size check mismatch 12 != 13") +} + func (s *DownloaderSuite) TestDownloadTempError(c *C) { d := NewDownloader(2) defer d.Shutdown() @@ -189,11 +207,17 @@ const ( func (s *DownloaderSuite) TestDownloadTryCompression(c *C) { var buf []byte + expectedChecksums := map[string]ChecksumInfo{ + "file.bz2": ChecksumInfo{Size: int64(len(bzipData))}, + "file.gz": ChecksumInfo{Size: int64(len(gzipData))}, + "file": ChecksumInfo{Size: int64(len(rawData))}, + } + // bzip2 only available buf = make([]byte, 4) d := NewFakeDownloader() d.ExpectResponse("http://example.com/file.bz2", bzipData) - r, file, err := DownloadTryCompression(d, "http://example.com/file") + r, file, err := DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false) c.Assert(err, IsNil) defer file.Close() io.ReadFull(r, buf) @@ -205,7 +229,7 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) { d = NewFakeDownloader() d.ExpectError("http://example.com/file.bz2", errors.New("404")) d.ExpectResponse("http://example.com/file.gz", gzipData) - r, file, err = DownloadTryCompression(d, "http://example.com/file") + r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false) c.Assert(err, IsNil) defer file.Close() io.ReadFull(r, buf) @@ -218,7 +242,7 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) { d.ExpectError("http://example.com/file.bz2", errors.New("404")) d.ExpectError("http://example.com/file.gz", errors.New("404")) d.ExpectResponse("http://example.com/file", rawData) - r, file, err = DownloadTryCompression(d, "http://example.com/file") + r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false) c.Assert(err, IsNil) defer file.Close() io.ReadFull(r, buf) @@ -231,7 +255,7 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) { d.ExpectError("http://example.com/file.bz2", errors.New("404")) d.ExpectResponse("http://example.com/file.gz", "x") d.ExpectResponse("http://example.com/file", "recovered") - r, file, err = DownloadTryCompression(d, "http://example.com/file") + r, file, err = DownloadTryCompression(d, "http://example.com/file", nil, false) c.Assert(err, IsNil) defer file.Close() io.ReadFull(r, buf) @@ -241,13 +265,20 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) { func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) { d := NewFakeDownloader() - _, _, err := DownloadTryCompression(d, "http://example.com/file") + _, _, err := DownloadTryCompression(d, "http://example.com/file", nil, false) c.Assert(err, ErrorMatches, "unexpected request.*") d = NewFakeDownloader() d.ExpectError("http://example.com/file.bz2", errors.New("404")) d.ExpectError("http://example.com/file.gz", errors.New("404")) d.ExpectError("http://example.com/file", errors.New("403")) - _, _, err = DownloadTryCompression(d, "http://example.com/file") + _, _, err = DownloadTryCompression(d, "http://example.com/file", nil, false) c.Assert(err, ErrorMatches, "403") + + d = NewFakeDownloader() + d.ExpectError("http://example.com/file.bz2", errors.New("404")) + d.ExpectError("http://example.com/file.gz", errors.New("404")) + d.ExpectResponse("http://example.com/file", rawData) + _, _, err = DownloadTryCompression(d, "http://example.com/file", map[string]ChecksumInfo{"file": ChecksumInfo{Size: 7}}, false) + c.Assert(err, ErrorMatches, "checksums don't match.*") } diff --git a/utils/fake.go b/utils/fake.go index 2a290071..eb6af4a1 100644 --- a/utils/fake.go +++ b/utils/fake.go @@ -49,7 +49,7 @@ func (f *FakeDownloader) Empty() bool { } // DownloadWithChecksum performs fake download by matching against first expectation in the queue, with cheksum verification -func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, result chan<- error, expected ChecksumInfo) { +func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, result chan<- error, expected ChecksumInfo, ignoreMismatch bool) { if len(f.expected) == 0 || f.expected[0].URL != url { result <- fmt.Errorf("unexpected request for %s", url) return @@ -85,10 +85,15 @@ func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, resul return } - if expected.MD5 != "" { - if expected != cks.Sum() { - result <- fmt.Errorf("checksums don't match: %#v != %#v", expected, cks.Sum()) - return + if expected.Size != -1 { + if expected.Size != cks.Sum().Size || expected.MD5 != "" && expected.MD5 != cks.Sum().MD5 || + expected.SHA1 != "" && expected.SHA1 != cks.Sum().SHA1 || expected.SHA256 != "" && expected.SHA256 != cks.Sum().SHA256 { + if ignoreMismatch { + fmt.Printf("WARNING: checksums don't match: %#v != %#v\n", expected, cks.Sum()) + } else { + result <- fmt.Errorf("checksums don't match: %#v != %#v", expected, cks.Sum()) + return + } } } @@ -98,7 +103,7 @@ func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, resul // Download performs fake download by matching against first expectation in the queue func (f *FakeDownloader) Download(url string, filename string, result chan<- error) { - f.DownloadWithChecksum(url, filename, result, ChecksumInfo{}) + f.DownloadWithChecksum(url, filename, result, ChecksumInfo{Size: -1}, false) } // Shutdown does nothing