Refactor to separate FakeDownloader, DownloadWithCompression, repo download.

utils/download.go

@@ -1,6 +1,8 @@
 package utils
 
 import (
+	"compress/bzip2"
+	"compress/gzip"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -12,7 +14,6 @@ import (
 // Downloader is parallel HTTP fetcher
 type Downloader interface {
 	Download(url string, destination string) <-chan error
-	DownloadTemp(url string) (*os.File, error)
 	Shutdown()
 }
 
@@ -74,30 +75,6 @@ func (downloader *downloaderImpl) Download(url string, destination string) <-cha
 	return ch
 }
 
-// DownloadTemp starts new download to temporary file and returns File
-//
-// Temporary file would be already removed, so no need to cleanup
-func (downloader *downloaderImpl) DownloadTemp(url string) (*os.File, error) {
-	ch := make(chan error, 1)
-
-	tempfile, err := ioutil.TempFile(os.TempDir(), "aptly")
-	if err != nil {
-		return nil, err
-	}
-
-	defer os.Remove(tempfile.Name())
-
-	downloader.queue <- &downloadTask{url: url, destination: tempfile.Name(), result: ch}
-
-	err = <-ch
-	if err != nil {
-		tempfile.Close()
-		return nil, err
-	}
-
-	return tempfile, nil
-}
-
 // handleTask processes single download task
 func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	log.Printf("Downloading %s...\n", task.url)
@@ -138,3 +115,66 @@ func (downloader *downloaderImpl) process() {
 		}
 	}
 }
+
+// DownloadTemp starts new download to temporary file and returns File
+//
+// Temporary file would be already removed, so no need to cleanup
+func DownloadTemp(downloader Downloader, url string) (*os.File, error) {
+
+	tempfile, err := ioutil.TempFile(os.TempDir(), "aptly")
+	if err != nil {
+		return nil, err
+	}
+
+	defer os.Remove(tempfile.Name())
+
+	ch := downloader.Download(url, tempfile.Name())
+
+	err = <-ch
+	if err != nil {
+		tempfile.Close()
+		return nil, err
+	}
+
+	return tempfile, nil
+}
+
+// List of extensions + corresponding uncompression support
+var compressionMethods = []struct {
+	extenstion     string
+	transformation func(io.Reader) (io.Reader, error)
+}{
+	{
+		extenstion:     ".bz2",
+		transformation: func(r io.Reader) (io.Reader, error) { return bzip2.NewReader(r), nil },
+	},
+	{
+		extenstion:     ".gz",
+		transformation: func(r io.Reader) (io.Reader, error) { return gzip.NewReader(r) },
+	},
+	{
+		extenstion:     "",
+		transformation: func(r io.Reader) (io.Reader, error) { return r, nil },
+	},
+}
+
+// DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until
+// it finds existing file.
+func DownloadTryCompression(downloader Downloader, url string) (io.Reader, *os.File, error) {
+	var err error
+
+	for _, method := range compressionMethods {
+		file, err := DownloadTemp(downloader, url+method.extenstion)
+		if err != nil {
+			continue
+		}
+
+		uncompressed, err := method.transformation(file)
+		if err != nil {
+			continue
+		}
+
+		return uncompressed, file, err
+	}
+	return nil, nil, err
+}

utils/download_test.go

@@ -79,7 +79,7 @@ func (s *DownloaderSuite) TestDownloadTemp(c *C) {
 	d := NewDownloader(2)
 	defer d.Shutdown()
 
-	f, err := d.DownloadTemp("http://smira.ru/")
+	f, err := DownloadTemp(d, "http://smira.ru/")
 	c.Assert(err, IsNil)
 	defer f.Close()
 
@@ -96,7 +96,7 @@ func (s *DownloaderSuite) TestDownloadTempError(c *C) {
 	d := NewDownloader(2)
 	defer d.Shutdown()
 
-	f, err := d.DownloadTemp("http://smira.ru/doesntexist")
+	f, err := DownloadTemp(d, "http://smira.ru/doesntexist")
 	c.Assert(err, NotNil)
 	c.Assert(f, IsNil)
 	c.Assert(err, ErrorMatches, "HTTP code 404.*")

utils/fake.go

@@ -0,0 +1,85 @@
+package utils
+
+import (
+	"fmt"
+	"os"
+)
+
+type expectedRequest struct {
+	Url      string
+	Err      error
+	Response string
+}
+
+// FakeDownloader is like Downloader, but it used in tests
+// to stub out results
+type FakeDownloader struct {
+	expected []expectedRequest
+}
+
+// Check interface
+var (
+	_ Downloader = &FakeDownloader{}
+)
+
+// NewFakeDownloader creates new expected downloader
+func NewFakeDownloader() *FakeDownloader {
+	result := &FakeDownloader{}
+	result.expected = make([]expectedRequest, 0)
+	return result
+}
+
+// ExpectResponse installs expectation on upcoming download with response
+func (f *FakeDownloader) ExpectResponse(url string, response string) *FakeDownloader {
+	f.expected = append(f.expected, expectedRequest{Url: url, Response: response})
+	return f
+}
+
+// ExpectError installs expectation on upcoming download with error
+func (f *FakeDownloader) ExpectError(url string, err error) *FakeDownloader {
+	f.expected = append(f.expected, expectedRequest{Url: url, Err: err})
+	return f
+}
+
+// Empty verifies that are planned downloads have happened
+func (f *FakeDownloader) Empty() bool {
+	return len(f.expected) == 0
+}
+
+// Download performs fake download by matching against first expectation in the queue
+func (f *FakeDownloader) Download(url string, filename string) <-chan error {
+	result := make(chan error, 1)
+
+	if len(f.expected) == 0 || f.expected[0].Url != url {
+		result <- fmt.Errorf("unexpected request for %s", url)
+		return result
+	}
+
+	expected := f.expected[0]
+	f.expected = f.expected[1:]
+
+	if expected.Err != nil {
+		result <- expected.Err
+		return result
+	}
+
+	outfile, err := os.Create(filename)
+	if err != nil {
+		result <- err
+		return result
+	}
+	defer outfile.Close()
+
+	_, err = outfile.Write([]byte(expected.Response))
+	if err != nil {
+		result <- err
+		return result
+	}
+
+	result <- nil
+	return result
+}
+
+// Shutdown does nothing
+func (f *FakeDownloader) Shutdown() {
+}