diff --git a/debian/list.go b/debian/list.go new file mode 100644 index 00000000..99f17519 --- /dev/null +++ b/debian/list.go @@ -0,0 +1,46 @@ +package debian + +import ( + "fmt" +) + +// PackageList is list of unique (by key) packages +// +// It could be seen as repo snapshot, repo contents, result of filtering, +// merge, etc. +type PackageList struct { + packages map[string]*Package +} + +// NewPackageList creates empty package list +func NewPackageList() *PackageList { + return &PackageList{packages: make(map[string]*Package, 1000)} +} + +// Add appends package to package list, additionally checking for uniqueness +func (l *PackageList) Add(p *Package) error { + key := string(p.Key()) + existing, ok := l.packages[key] + if ok { + if !existing.Equals(p) { + return fmt.Errorf("conflict in package %s: %#v != %#v", p, existing, p) + } + return nil + } + l.packages[key] = p + return nil +} + +// ForEach calls handler for each package in list +// +// TODO: Error handling +func (l *PackageList) ForEach(handler func(*Package)) { + for _, p := range l.packages { + handler(p) + } +} + +// Length returns number of packages in the list +func (l *PackageList) Length() int { + return len(l.packages) +} diff --git a/debian/list_test.go b/debian/list_test.go new file mode 100644 index 00000000..9cdb2a98 --- /dev/null +++ b/debian/list_test.go @@ -0,0 +1,57 @@ +package debian + +import ( + debc "github.com/smira/godebiancontrol" + . "launchpad.net/gocheck" +) + +type PackageListSuite struct { + list *PackageList + p1, p2, p3, p4 *Package +} + +var _ = Suite(&PackageListSuite{}) + +func (s *PackageListSuite) SetUpTest(c *C) { + s.list = NewPackageList() + + paraGen := func() debc.Paragraph { + para := make(debc.Paragraph) + for k, v := range packagePara { + para[k] = v + } + return para + } + + s.p1 = NewPackageFromControlFile(paraGen()) + s.p2 = NewPackageFromControlFile(paraGen()) + para := paraGen() + para["Package"] = "mars-invaders" + s.p3 = NewPackageFromControlFile(para) + para = paraGen() + para["Size"] = "42" + s.p4 = NewPackageFromControlFile(para) +} + +func (s *PackageListSuite) TestAddLength(c *C) { + c.Check(s.list.Length(), Equals, 0) + c.Check(s.list.Add(s.p1), IsNil) + c.Check(s.list.Length(), Equals, 1) + c.Check(s.list.Add(s.p2), IsNil) + c.Check(s.list.Length(), Equals, 1) + c.Check(s.list.Add(s.p3), IsNil) + c.Check(s.list.Length(), Equals, 2) + c.Check(s.list.Add(s.p4), ErrorMatches, "conflict in package.*") +} + +func (s *PackageListSuite) TestForeach(c *C) { + s.list.Add(s.p1) + s.list.Add(s.p3) + + length := 0 + s.list.ForEach(func(*Package) { + length++ + }) + + c.Check(length, Equals, 2) +} diff --git a/debian/package.go b/debian/package.go index 42e4b39d..8558c5ea 100644 --- a/debian/package.go +++ b/debian/package.go @@ -6,6 +6,8 @@ import ( "github.com/smira/aptly/utils" debc "github.com/smira/godebiancontrol" "github.com/ugorji/go/codec" + "os" + "strconv" "strings" ) @@ -16,6 +18,7 @@ type Package struct { Name string Version string Filename string + Filesize int64 Architecture string Depends []string PreDepends []string @@ -49,6 +52,9 @@ func NewPackageFromControlFile(input debc.Paragraph) *Package { delete(input, "Filename") delete(input, "Architecture") + result.Filesize, _ = strconv.ParseInt(input["Size"], 10, 64) + delete(input, "Size") + result.Depends = parseDependencies(input, "Depends") result.PreDepends = parseDependencies(input, "Pre-Depends") result.Suggests = parseDependencies(input, "Suggests") @@ -90,5 +96,15 @@ func (p *Package) Equals(p2 *Package) bool { return p.Name == p2.Name && p.Version == p2.Version && p.Filename == p2.Filename && p.Architecture == p2.Architecture && utils.StrSlicesEqual(p.Depends, p2.Depends) && utils.StrSlicesEqual(p.PreDepends, p2.PreDepends) && utils.StrSlicesEqual(p.Suggests, p2.Suggests) && - utils.StrSlicesEqual(p.Recommends, p2.Recommends) && utils.StrMapsEqual(p.Extra, p2.Extra) + utils.StrSlicesEqual(p.Recommends, p2.Recommends) && utils.StrMapsEqual(p.Extra, p2.Extra) && + p.Filesize == p2.Filesize +} + +// VerifyFile verifies integrity and existence of local files for the package +func (p *Package) VerifyFile(filepath string) bool { + st, err := os.Stat(filepath) + if err != nil { + return false + } + return st.Size() == p.Filesize } diff --git a/debian/package_test.go b/debian/package_test.go index f47a6b83..ec4c3cb0 100644 --- a/debian/package_test.go +++ b/debian/package_test.go @@ -29,6 +29,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) { c.Check(p.Filename, Equals, "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb") c.Check(p.Depends, DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)"}) c.Check(p.Suggests, IsNil) + c.Check(p.Filesize, Equals, int64(187518)) } func (s *PackageSuite) TestKey(c *C) { diff --git a/debian/remote.go b/debian/remote.go index 6986bfe3..76d5f064 100644 --- a/debian/remote.go +++ b/debian/remote.go @@ -60,6 +60,13 @@ func (repo *RemoteRepo) BinaryURL(component string, architecture string) *url.UR return repo.archiveRootURL.ResolveReference(path) } +// PackageURL returns URL of package file relative to repository root +// architecture +func (repo *RemoteRepo) PackageURL(filename string) *url.URL { + path := &url.URL{Path: filename} + return repo.archiveRootURL.ResolveReference(path) +} + // Fetch updates information about repository func (repo *RemoteRepo) Fetch(d utils.Downloader) error { // Download release file to temporary URL @@ -106,7 +113,10 @@ func (repo *RemoteRepo) Fetch(d utils.Downloader) error { } // Download downloads all repo files -func (repo *RemoteRepo) Download(d utils.Downloader, db database.Storage) error { +func (repo *RemoteRepo) Download(d utils.Downloader, db database.Storage, packageRepo *Repository) error { + list := NewPackageList() + + // Download and parse all Release files for _, component := range repo.Components { for _, architecture := range repo.Architectures { packagesReader, packagesFile, err := utils.DownloadTryCompression(d, repo.BinaryURL(component, architecture).String()) @@ -122,10 +132,37 @@ func (repo *RemoteRepo) Download(d utils.Downloader, db database.Storage) error for _, para := range paras { p := NewPackageFromControlFile(para) - db.Put(p.Key(), p.Encode()) + + list.Add(p) } } } + // Save package meta information to DB + list.ForEach(func(p *Package) { + db.Put(p.Key(), p.Encode()) + }) + + // Download all package files + ch := make(chan error, list.Length()) + count := 0 + + list.ForEach(func(p *Package) { + poolPath, err := packageRepo.PoolPath(p.Filename) + if err == nil { + if !p.VerifyFile(poolPath) { + d.Download(repo.PackageURL(p.Filename).String(), poolPath, ch) + count++ + } + } + }) + + // Wait for all downloads to finish + // TODO: report errors + for count > 0 { + _ = <-ch + count-- + } + return nil }