Refactor to separate FakeDownloader, DownloadWithCompression, repo download.

debian/remote.go

@@ -3,6 +3,7 @@ package debian
 
 import (
 	"fmt"
+	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/utils"
 	debc "github.com/smira/godebiancontrol"
 	"net/url"
@@ -52,10 +53,17 @@ func (repo *RemoteRepo) ReleaseURL() *url.URL {
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
+// BinaryURL returns URL of Packages file for given component and
+// architecture
+func (repo *RemoteRepo) BinaryURL(component string, architecture string) *url.URL {
+	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/binary-%s/Packages", repo.Distribution, component, architecture)}
+	return repo.archiveRootURL.ResolveReference(path)
+}
+
 // Fetch updates information about repository
 func (repo *RemoteRepo) Fetch(d utils.Downloader) error {
 	// Download release file to temporary URL
-	release, err := d.DownloadTemp(repo.ReleaseURL().String())
+	release, err := utils.DownloadTemp(d, repo.ReleaseURL().String())
 	if err != nil {
 		return err
 	}
@@ -96,3 +104,28 @@ func (repo *RemoteRepo) Fetch(d utils.Downloader) error {
 
 	return nil
 }
+
+// Download downloads all repo files
+func (repo *RemoteRepo) Download(d utils.Downloader, db database.Storage) error {
+	for _, component := range repo.Components {
+		for _, architecture := range repo.Architectures {
+			packagesReader, packagesFile, err := utils.DownloadTryCompression(d, repo.BinaryURL(component, architecture).String())
+			if err != nil {
+				return err
+			}
+			defer packagesFile.Close()
+
+			paras, err := debc.Parse(packagesReader)
+			if err != nil {
+				return err
+			}
+
+			for _, para := range paras {
+				p := NewPackageFromControlFile(para)
+				db.Put(p.Key(), p.Encode())
+			}
+		}
+	}
+
+	return nil
+}

debian/remote_test.go

@@ -2,46 +2,10 @@ package debian
 
 import (
 	"github.com/smira/aptly/utils"
-	"io/ioutil"
 	. "launchpad.net/gocheck"
-	"os"
 	"testing"
 )
 
-type FakeDownloader struct {
-	Err      error
-	Response string
-}
-
-func (f *FakeDownloader) DownloadTemp(url string) (*os.File, error) {
-	if f.Err != nil {
-		return nil, f.Err
-	}
-
-	tempfile, _ := ioutil.TempFile(os.TempDir(), "aptly-test")
-	defer os.Remove(tempfile.Name())
-
-	tempfile.Write([]byte(f.Response))
-	tempfile.Seek(0, 0)
-
-	return tempfile, nil
-}
-
-func (f *FakeDownloader) Download(url string, filename string) <-chan error {
-	result := make(chan error)
-	if f.Err != nil {
-		result <- f.Err
-		return result
-	}
-
-	// TODO
-
-	return result
-}
-
-func (f *FakeDownloader) Shutdown() {
-}
-
 // Launch gocheck tests
 func Test(t *testing.T) {
 	TestingT(t)
@@ -56,7 +20,7 @@ var _ = Suite(&RemoteRepoSuite{})
 
 func (s *RemoteRepoSuite) SetUpTest(c *C) {
 	s.repo, _ = NewRemoteRepo("http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{})
-	s.downloader = &FakeDownloader{Response: exampleReleaseFile}
+	s.downloader = utils.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 }
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
@@ -64,6 +28,14 @@ func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
 	c.Assert(err, ErrorMatches, ".*hexadecimal escape in host.*")
 }
 
+func (s *RemoteRepoSuite) TestReleaseURL(c *C) {
+	c.Assert(s.repo.ReleaseURL().String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/Release")
+}
+
+func (s *RemoteRepoSuite) TestBinaryURL(c *C) {
+	c.Assert(s.repo.BinaryURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/binary-amd64/Packages")
+}
+
 func (s *RemoteRepoSuite) TestFetch(c *C) {
 	err := s.repo.Fetch(s.downloader)
 	c.Assert(err, IsNil)

utils/download.go

@@ -1,6 +1,8 @@
 package utils
 
 import (
+	"compress/bzip2"
+	"compress/gzip"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -12,7 +14,6 @@ import (
 // Downloader is parallel HTTP fetcher
 type Downloader interface {
 	Download(url string, destination string) <-chan error
-	DownloadTemp(url string) (*os.File, error)
 	Shutdown()
 }
 
@@ -74,30 +75,6 @@ func (downloader *downloaderImpl) Download(url string, destination string) <-cha
 	return ch
 }
 
-// DownloadTemp starts new download to temporary file and returns File
-//
-// Temporary file would be already removed, so no need to cleanup
-func (downloader *downloaderImpl) DownloadTemp(url string) (*os.File, error) {
-	ch := make(chan error, 1)
-
-	tempfile, err := ioutil.TempFile(os.TempDir(), "aptly")
-	if err != nil {
-		return nil, err
-	}
-
-	defer os.Remove(tempfile.Name())
-
-	downloader.queue <- &downloadTask{url: url, destination: tempfile.Name(), result: ch}
-
-	err = <-ch
-	if err != nil {
-		tempfile.Close()
-		return nil, err
-	}
-
-	return tempfile, nil
-}
-
 // handleTask processes single download task
 func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	log.Printf("Downloading %s...\n", task.url)
@@ -138,3 +115,66 @@ func (downloader *downloaderImpl) process() {
 		}
 	}
 }
+
+// DownloadTemp starts new download to temporary file and returns File
+//
+// Temporary file would be already removed, so no need to cleanup
+func DownloadTemp(downloader Downloader, url string) (*os.File, error) {
+
+	tempfile, err := ioutil.TempFile(os.TempDir(), "aptly")
+	if err != nil {
+		return nil, err
+	}
+
+	defer os.Remove(tempfile.Name())
+
+	ch := downloader.Download(url, tempfile.Name())
+
+	err = <-ch
+	if err != nil {
+		tempfile.Close()
+		return nil, err
+	}
+
+	return tempfile, nil
+}
+
+// List of extensions + corresponding uncompression support
+var compressionMethods = []struct {
+	extenstion     string
+	transformation func(io.Reader) (io.Reader, error)
+}{
+	{
+		extenstion:     ".bz2",
+		transformation: func(r io.Reader) (io.Reader, error) { return bzip2.NewReader(r), nil },
+	},
+	{
+		extenstion:     ".gz",
+		transformation: func(r io.Reader) (io.Reader, error) { return gzip.NewReader(r) },
+	},
+	{
+		extenstion:     "",
+		transformation: func(r io.Reader) (io.Reader, error) { return r, nil },
+	},
+}
+
+// DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until
+// it finds existing file.
+func DownloadTryCompression(downloader Downloader, url string) (io.Reader, *os.File, error) {
+	var err error
+
+	for _, method := range compressionMethods {
+		file, err := DownloadTemp(downloader, url+method.extenstion)
+		if err != nil {
+			continue
+		}
+
+		uncompressed, err := method.transformation(file)
+		if err != nil {
+			continue
+		}
+
+		return uncompressed, file, err
+	}
+	return nil, nil, err
+}

utils/download_test.go

@@ -79,7 +79,7 @@ func (s *DownloaderSuite) TestDownloadTemp(c *C) {
 	d := NewDownloader(2)
 	defer d.Shutdown()
 
-	f, err := d.DownloadTemp("http://smira.ru/")
+	f, err := DownloadTemp(d, "http://smira.ru/")
 	c.Assert(err, IsNil)
 	defer f.Close()
 
@@ -96,7 +96,7 @@ func (s *DownloaderSuite) TestDownloadTempError(c *C) {
 	d := NewDownloader(2)
 	defer d.Shutdown()
 
-	f, err := d.DownloadTemp("http://smira.ru/doesntexist")
+	f, err := DownloadTemp(d, "http://smira.ru/doesntexist")
 	c.Assert(err, NotNil)
 	c.Assert(f, IsNil)
 	c.Assert(err, ErrorMatches, "HTTP code 404.*")

utils/fake.go

@@ -0,0 +1,85 @@
+package utils
+
+import (
+	"fmt"
+	"os"
+)
+
+type expectedRequest struct {
+	Url      string
+	Err      error
+	Response string
+}
+
+// FakeDownloader is like Downloader, but it used in tests
+// to stub out results
+type FakeDownloader struct {
+	expected []expectedRequest
+}
+
+// Check interface
+var (
+	_ Downloader = &FakeDownloader{}
+)
+
+// NewFakeDownloader creates new expected downloader
+func NewFakeDownloader() *FakeDownloader {
+	result := &FakeDownloader{}
+	result.expected = make([]expectedRequest, 0)
+	return result
+}
+
+// ExpectResponse installs expectation on upcoming download with response
+func (f *FakeDownloader) ExpectResponse(url string, response string) *FakeDownloader {
+	f.expected = append(f.expected, expectedRequest{Url: url, Response: response})
+	return f
+}
+
+// ExpectError installs expectation on upcoming download with error
+func (f *FakeDownloader) ExpectError(url string, err error) *FakeDownloader {
+	f.expected = append(f.expected, expectedRequest{Url: url, Err: err})
+	return f
+}
+
+// Empty verifies that are planned downloads have happened
+func (f *FakeDownloader) Empty() bool {
+	return len(f.expected) == 0
+}
+
+// Download performs fake download by matching against first expectation in the queue
+func (f *FakeDownloader) Download(url string, filename string) <-chan error {
+	result := make(chan error, 1)
+
+	if len(f.expected) == 0 || f.expected[0].Url != url {
+		result <- fmt.Errorf("unexpected request for %s", url)
+		return result
+	}
+
+	expected := f.expected[0]
+	f.expected = f.expected[1:]
+
+	if expected.Err != nil {
+		result <- expected.Err
+		return result
+	}
+
+	outfile, err := os.Create(filename)
+	if err != nil {
+		result <- err
+		return result
+	}
+	defer outfile.Close()
+
+	_, err = outfile.Write([]byte(expected.Response))
+	if err != nil {
+		result <- err
+		return result
+	}
+
+	result <- nil
+	return result
+}
+
+// Shutdown does nothing
+func (f *FakeDownloader) Shutdown() {
+}