Repository mirroring: working first version.

debian/list.go

@@ -0,0 +1,46 @@
+package debian
+
+import (
+	"fmt"
+)
+
+// PackageList is list of unique (by key) packages
+//
+// It could be seen as repo snapshot, repo contents, result of filtering,
+// merge, etc.
+type PackageList struct {
+	packages map[string]*Package
+}
+
+// NewPackageList creates empty package list
+func NewPackageList() *PackageList {
+	return &PackageList{packages: make(map[string]*Package, 1000)}
+}
+
+// Add appends package to package list, additionally checking for uniqueness
+func (l *PackageList) Add(p *Package) error {
+	key := string(p.Key())
+	existing, ok := l.packages[key]
+	if ok {
+		if !existing.Equals(p) {
+			return fmt.Errorf("conflict in package %s: %#v != %#v", p, existing, p)
+		}
+		return nil
+	}
+	l.packages[key] = p
+	return nil
+}
+
+// ForEach calls handler for each package in list
+//
+// TODO: Error handling
+func (l *PackageList) ForEach(handler func(*Package)) {
+	for _, p := range l.packages {
+		handler(p)
+	}
+}
+
+// Length returns number of packages in the list
+func (l *PackageList) Length() int {
+	return len(l.packages)
+}

debian/list_test.go

@@ -0,0 +1,57 @@
+package debian
+
+import (
+	debc "github.com/smira/godebiancontrol"
+	. "launchpad.net/gocheck"
+)
+
+type PackageListSuite struct {
+	list           *PackageList
+	p1, p2, p3, p4 *Package
+}
+
+var _ = Suite(&PackageListSuite{})
+
+func (s *PackageListSuite) SetUpTest(c *C) {
+	s.list = NewPackageList()
+
+	paraGen := func() debc.Paragraph {
+		para := make(debc.Paragraph)
+		for k, v := range packagePara {
+			para[k] = v
+		}
+		return para
+	}
+
+	s.p1 = NewPackageFromControlFile(paraGen())
+	s.p2 = NewPackageFromControlFile(paraGen())
+	para := paraGen()
+	para["Package"] = "mars-invaders"
+	s.p3 = NewPackageFromControlFile(para)
+	para = paraGen()
+	para["Size"] = "42"
+	s.p4 = NewPackageFromControlFile(para)
+}
+
+func (s *PackageListSuite) TestAddLength(c *C) {
+	c.Check(s.list.Length(), Equals, 0)
+	c.Check(s.list.Add(s.p1), IsNil)
+	c.Check(s.list.Length(), Equals, 1)
+	c.Check(s.list.Add(s.p2), IsNil)
+	c.Check(s.list.Length(), Equals, 1)
+	c.Check(s.list.Add(s.p3), IsNil)
+	c.Check(s.list.Length(), Equals, 2)
+	c.Check(s.list.Add(s.p4), ErrorMatches, "conflict in package.*")
+}
+
+func (s *PackageListSuite) TestForeach(c *C) {
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+
+	length := 0
+	s.list.ForEach(func(*Package) {
+		length++
+	})
+
+	c.Check(length, Equals, 2)
+}

debian/package.go

@@ -6,6 +6,8 @@ import (
 	"github.com/smira/aptly/utils"
 	debc "github.com/smira/godebiancontrol"
 	"github.com/ugorji/go/codec"
+	"os"
+	"strconv"
 	"strings"
 )
 
@@ -16,6 +18,7 @@ type Package struct {
 	Name         string
 	Version      string
 	Filename     string
+	Filesize     int64
 	Architecture string
 	Depends      []string
 	PreDepends   []string
@@ -49,6 +52,9 @@ func NewPackageFromControlFile(input debc.Paragraph) *Package {
 	delete(input, "Filename")
 	delete(input, "Architecture")
 
+	result.Filesize, _ = strconv.ParseInt(input["Size"], 10, 64)
+	delete(input, "Size")
+
 	result.Depends = parseDependencies(input, "Depends")
 	result.PreDepends = parseDependencies(input, "Pre-Depends")
 	result.Suggests = parseDependencies(input, "Suggests")
@@ -90,5 +96,15 @@ func (p *Package) Equals(p2 *Package) bool {
 	return p.Name == p2.Name && p.Version == p2.Version && p.Filename == p2.Filename &&
 		p.Architecture == p2.Architecture && utils.StrSlicesEqual(p.Depends, p2.Depends) &&
 		utils.StrSlicesEqual(p.PreDepends, p2.PreDepends) && utils.StrSlicesEqual(p.Suggests, p2.Suggests) &&
-		utils.StrSlicesEqual(p.Recommends, p2.Recommends) && utils.StrMapsEqual(p.Extra, p2.Extra)
+		utils.StrSlicesEqual(p.Recommends, p2.Recommends) && utils.StrMapsEqual(p.Extra, p2.Extra) &&
+		p.Filesize == p2.Filesize
+}
+
+// VerifyFile verifies integrity and existence of local files for the package
+func (p *Package) VerifyFile(filepath string) bool {
+	st, err := os.Stat(filepath)
+	if err != nil {
+		return false
+	}
+	return st.Size() == p.Filesize
 }

debian/package_test.go

@@ -29,6 +29,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 	c.Check(p.Filename, Equals, "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
 	c.Check(p.Depends, DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)"})
 	c.Check(p.Suggests, IsNil)
+	c.Check(p.Filesize, Equals, int64(187518))
 }
 
 func (s *PackageSuite) TestKey(c *C) {

debian/remote.go

@@ -60,6 +60,13 @@ func (repo *RemoteRepo) BinaryURL(component string, architecture string) *url.UR
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
+// PackageURL returns URL of package file relative to repository root
+// architecture
+func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
+	path := &url.URL{Path: filename}
+	return repo.archiveRootURL.ResolveReference(path)
+}
+
 // Fetch updates information about repository
 func (repo *RemoteRepo) Fetch(d utils.Downloader) error {
 	// Download release file to temporary URL
@@ -106,7 +113,10 @@ func (repo *RemoteRepo) Fetch(d utils.Downloader) error {
 }
 
 // Download downloads all repo files
-func (repo *RemoteRepo) Download(d utils.Downloader, db database.Storage) error {
+func (repo *RemoteRepo) Download(d utils.Downloader, db database.Storage, packageRepo *Repository) error {
+	list := NewPackageList()
+
+	// Download and parse all Release files
 	for _, component := range repo.Components {
 		for _, architecture := range repo.Architectures {
 			packagesReader, packagesFile, err := utils.DownloadTryCompression(d, repo.BinaryURL(component, architecture).String())
@@ -122,10 +132,37 @@ func (repo *RemoteRepo) Download(d utils.Downloader, db database.Storage) error
 
 			for _, para := range paras {
 				p := NewPackageFromControlFile(para)
-				db.Put(p.Key(), p.Encode())
+
+				list.Add(p)
 			}
 		}
 	}
 
+	// Save package meta information to DB
+	list.ForEach(func(p *Package) {
+		db.Put(p.Key(), p.Encode())
+	})
+
+	// Download all package files
+	ch := make(chan error, list.Length())
+	count := 0
+
+	list.ForEach(func(p *Package) {
+		poolPath, err := packageRepo.PoolPath(p.Filename)
+		if err == nil {
+			if !p.VerifyFile(poolPath) {
+				d.Download(repo.PackageURL(p.Filename).String(), poolPath, ch)
+				count++
+			}
+		}
+	})
+
+	// Wait for all downloads to finish
+	// TODO: report errors
+	for count > 0 {
+		_ = <-ch
+		count--
+	}
+
 	return nil
 }