From 3fe8a09928ae614fb1e1dec3a05182226e53a58f Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Tue, 4 Mar 2014 16:49:26 +0400 Subject: [PATCH] Major refactoring to lower memory consumption. Package has been split into multiple structure loaded on demand: files, extra, depends. They're saved and loaded completely separately. --- debian/list.go | 285 +--------------------- debian/list_test.go | 278 ++-------------------- debian/package.go | 377 ++++++++++-------------------- debian/package_collection.go | 256 ++++++++++++++++++++ debian/package_collection_test.go | 93 ++++++++ debian/package_deps.go | 30 +++ debian/package_files.go | 78 +++++++ debian/package_files_test.go | 61 +++++ debian/package_test.go | 236 +++++-------------- debian/publish.go | 4 + debian/publish_test.go | 2 +- debian/reflist.go | 286 +++++++++++++++++++++++ debian/reflist_test.go | 275 ++++++++++++++++++++++ debian/remote.go | 41 +--- debian/remote_test.go | 6 +- 15 files changed, 1301 insertions(+), 1007 deletions(-) create mode 100644 debian/package_collection.go create mode 100644 debian/package_collection_test.go create mode 100644 debian/package_deps.go create mode 100644 debian/package_files.go create mode 100644 debian/package_files_test.go create mode 100644 debian/reflist.go create mode 100644 debian/reflist_test.go diff --git a/debian/list.go b/debian/list.go index 982b6364..6eeab298 100644 --- a/debian/list.go +++ b/debian/list.go @@ -1,10 +1,8 @@ package debian import ( - "bytes" "fmt" "github.com/smira/aptly/utils" - "github.com/ugorji/go/codec" "sort" "strings" ) @@ -76,7 +74,7 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle // Add appends package to package list, additionally checking for uniqueness func (l *PackageList) Add(p *Package) error { - key := string(p.Key()) + key := string(p.Key("")) existing, ok := l.packages[key] if ok { if !existing.Equals(p) { @@ -139,7 +137,7 @@ func (l *PackageList) Append(pl *PackageList) error { // Remove removes package from the list, and updates index when required func (l *PackageList) Remove(p *Package) { - delete(l.packages, string(p.Key())) + delete(l.packages, string(p.Key(""))) if l.indexed { for _, provides := range p.Provides { for i, pkg := range l.providesIndex[provides] { @@ -413,282 +411,3 @@ func (l *PackageList) Filter(queries []string, withDependencies bool, source *Pa return result, nil } - -// PackageRefList is a list of keys of packages, this is basis for snapshot -// and similar stuff -// -// Refs are sorted in lexicographical order -type PackageRefList struct { - // List of package keys - Refs [][]byte -} - -// Verify interface -var ( - _ sort.Interface = &PackageRefList{} -) - -// NewPackageRefList creates empty PackageRefList -func NewPackageRefList() *PackageRefList { - return &PackageRefList{} -} - -// NewPackageRefListFromPackageList creates PackageRefList from PackageList -func NewPackageRefListFromPackageList(list *PackageList) *PackageRefList { - reflist := &PackageRefList{} - reflist.Refs = make([][]byte, list.Len()) - - i := 0 - for _, p := range list.packages { - reflist.Refs[i] = p.Key() - i++ - } - - sort.Sort(reflist) - - return reflist -} - -// Len returns number of refs -func (l *PackageRefList) Len() int { - return len(l.Refs) -} - -// Swap swaps two refs -func (l *PackageRefList) Swap(i, j int) { - l.Refs[i], l.Refs[j] = l.Refs[j], l.Refs[i] -} - -// Compare compares two refs in lexographical order -func (l *PackageRefList) Less(i, j int) bool { - return bytes.Compare(l.Refs[i], l.Refs[j]) < 0 -} - -// Encode does msgpack encoding of PackageRefList -func (l *PackageRefList) Encode() []byte { - var buf bytes.Buffer - - encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{}) - encoder.Encode(l) - - return buf.Bytes() -} - -// Decode decodes msgpack representation into PackageRefLit -func (l *PackageRefList) Decode(input []byte) error { - decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{}) - return decoder.Decode(l) -} - -// ForEach calls handler for each package ref in list -func (l *PackageRefList) ForEach(handler func([]byte) error) error { - var err error - for _, p := range l.Refs { - err = handler(p) - if err != nil { - return err - } - } - return err -} - -// Substract returns all packages in l that are not in r -func (l *PackageRefList) Substract(r *PackageRefList) *PackageRefList { - result := &PackageRefList{Refs: make([][]byte, 0, 128)} - - // pointer to left and right reflists - il, ir := 0, 0 - // length of reflists - ll, lr := l.Len(), r.Len() - - for il < ll || ir < lr { - if il == ll { - // left list exhausted, we got the result - break - } - if ir == lr { - // right list exhausted, append what is left to result - result.Refs = append(result.Refs, l.Refs[il:]...) - break - } - - rel := bytes.Compare(l.Refs[il], r.Refs[ir]) - if rel == 0 { - // r contains entry from l, so we skip it - il++ - ir++ - } else if rel < 0 { - // item il is not in r, append - result.Refs = append(result.Refs, l.Refs[il]) - il++ - } else { - // skip over to next item in r - ir++ - } - } - - return result -} - -// PackageDiff is a difference between two packages in a list. -// -// If left & right are present, difference is in package version -// If left is nil, package is present only in right -// If right is nil, package is present only in left -type PackageDiff struct { - Left, Right *Package -} - -// PackageDiffs is a list of PackageDiff records -type PackageDiffs []PackageDiff - -// Diff calculates difference between two reflists -func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageCollection) (result PackageDiffs, err error) { - result = make(PackageDiffs, 0, 128) - - // pointer to left and right reflists - il, ir := 0, 0 - // length of reflists - ll, lr := l.Len(), r.Len() - // cached loaded packages on the left & right - pl, pr := (*Package)(nil), (*Package)(nil) - - // until we reached end of both lists - for il < ll || ir < lr { - // if we've exhausted left list, pull the rest from the right - if il == ll { - pr, err = packageCollection.ByKey(r.Refs[ir]) - if err != nil { - return nil, err - } - result = append(result, PackageDiff{Left: nil, Right: pr}) - ir++ - continue - } - // if we've exhausted right list, pull the rest from the left - if ir == lr { - pl, err = packageCollection.ByKey(l.Refs[il]) - if err != nil { - return nil, err - } - result = append(result, PackageDiff{Left: pl, Right: nil}) - il++ - continue - } - - // refs on both sides are present, load them - rl, rr := l.Refs[il], r.Refs[ir] - // compare refs - rel := bytes.Compare(rl, rr) - - if rel == 0 { - // refs are identical, so are packages, advance pointer - il++ - ir++ - pl, pr = nil, nil - } else { - // load pl & pr if they haven't been loaded before - if pl == nil { - pl, err = packageCollection.ByKey(rl) - if err != nil { - return nil, err - } - } - - if pr == nil { - pr, err = packageCollection.ByKey(rr) - if err != nil { - return nil, err - } - } - - // is pl & pr the same package, but different version? - if pl.Name == pr.Name && pl.Architecture == pr.Architecture { - result = append(result, PackageDiff{Left: pl, Right: pr}) - il++ - ir++ - pl, pr = nil, nil - } else { - // otherwise pl or pr is missing on one of the sides - if rel < 0 { - result = append(result, PackageDiff{Left: pl, Right: nil}) - il++ - pl = nil - } else { - result = append(result, PackageDiff{Left: nil, Right: pr}) - ir++ - pr = nil - } - } - - } - } - - return -} - -// Merge merges reflist r into current reflist. If overrideMatching, merge replaces matching packages (by architecture/name) -// with reference from r, otherwise all packages are saved. -func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result *PackageRefList) { - // pointer to left and right reflists - il, ir := 0, 0 - // length of reflists - ll, lr := l.Len(), r.Len() - - result = &PackageRefList{} - result.Refs = make([][]byte, 0, ll+lr) - - // until we reached end of both lists - for il < ll || ir < lr { - // if we've exhausted left list, pull the rest from the right - if il == ll { - result.Refs = append(result.Refs, r.Refs[ir:]...) - break - } - // if we've exhausted right list, pull the rest from the left - if ir == lr { - result.Refs = append(result.Refs, l.Refs[il:]...) - break - } - - // refs on both sides are present, load them - rl, rr := l.Refs[il], r.Refs[ir] - // compare refs - rel := bytes.Compare(rl, rr) - - if rel == 0 { - // refs are identical, so are packages, advance pointer - result.Refs = append(result.Refs, l.Refs[il]) - il++ - ir++ - } else { - if overrideMatching { - partsL := bytes.Split(rl, []byte(" ")) - archL, nameL := partsL[0][1:], partsL[1] - - partsR := bytes.Split(rr, []byte(" ")) - archR, nameR := partsR[0][1:], partsR[1] - - if bytes.Compare(archL, archR) == 0 && bytes.Compare(nameL, nameR) == 0 { - // override with package from the right - result.Refs = append(result.Refs, r.Refs[ir]) - il++ - ir++ - continue - } - } - - // otherwise append smallest of two - if rel < 0 { - result.Refs = append(result.Refs, l.Refs[il]) - il++ - } else { - result.Refs = append(result.Refs, r.Refs[ir]) - ir++ - } - - } - } - - return -} diff --git a/debian/list_test.go b/debian/list_test.go index bc3c4532..eab60bc9 100644 --- a/debian/list_test.go +++ b/debian/list_test.go @@ -2,7 +2,6 @@ package debian import ( "errors" - "github.com/smira/aptly/database" . "launchpad.net/gocheck" "sort" "strings" @@ -41,20 +40,20 @@ func (s *PackageListSuite) SetUpTest(c *C) { s.il = NewPackageList() s.packages = []*Package{ - &Package{Name: "lib", Version: "1.0", Architecture: "i386", Source: "lib (0.9)", PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"mail-agent"}}, - &Package{Name: "dpkg", Version: "1.7", Architecture: "i386", Provides: []string{"package-installer"}}, - &Package{Name: "data", Version: "1.1~bp1", Architecture: "all", Source: "app", PreDepends: []string{"dpkg (>= 1.6)"}}, - &Package{Name: "app", Version: "1.1~bp1", Architecture: "i386", PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}, - &Package{Name: "mailer", Version: "3.5.8", Architecture: "i386", Source: "postfix (1.3)", Provides: []string{"mail-agent"}}, - &Package{Name: "app", Version: "1.1~bp1", Architecture: "amd64", PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}, - &Package{Name: "app", Version: "1.1~bp1", Architecture: "arm", PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}, - &Package{Name: "app", Version: "1.0", Architecture: "s390", PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}, - &Package{Name: "aa", Version: "2.0-1", Architecture: "i386", PreDepends: []string{"dpkg (>= 1.6)"}}, - &Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "amd64", Provides: []string{"package-installer"}}, - &Package{Name: "libx", Version: "1.5", Architecture: "arm", PreDepends: []string{"dpkg (>= 1.6)"}}, - &Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "arm", Provides: []string{"package-installer"}}, - &Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "source", SourceArchitecture: "any", IsSource: true}, - &Package{Name: "dpkg", Version: "1.7", Architecture: "source", SourceArchitecture: "any", IsSource: true}, + &Package{Name: "lib", Version: "1.0", Architecture: "i386", Source: "lib (0.9)", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"mail-agent"}}}, + &Package{Name: "dpkg", Version: "1.7", Architecture: "i386", Provides: []string{"package-installer"}, deps: &PackageDependencies{}}, + &Package{Name: "data", Version: "1.1~bp1", Architecture: "all", Source: "app", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}}, + &Package{Name: "app", Version: "1.1~bp1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}}, + &Package{Name: "mailer", Version: "3.5.8", Architecture: "i386", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}}, + &Package{Name: "app", Version: "1.1~bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}}, + &Package{Name: "app", Version: "1.1~bp1", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}}, + &Package{Name: "app", Version: "1.0", Architecture: "s390", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}}, + &Package{Name: "aa", Version: "2.0-1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}}, + &Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "amd64", Provides: []string{"package-installer"}, deps: &PackageDependencies{}}, + &Package{Name: "libx", Version: "1.5", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}}, + &Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "arm", Provides: []string{"package-installer"}, deps: &PackageDependencies{}}, + &Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}}, + &Package{Name: "dpkg", Version: "1.7", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}}, } for _, p := range s.packages { s.il.Add(p) @@ -62,10 +61,10 @@ func (s *PackageListSuite) SetUpTest(c *C) { s.il.PrepareIndex() s.sourcePackages = []*Package{ - &Package{Name: "postfix", Version: "1.3", Architecture: "source", SourceArchitecture: "any", IsSource: true}, - &Package{Name: "app", Version: "1.1~bp1", Architecture: "source", SourceArchitecture: "any", IsSource: true}, - &Package{Name: "aa", Version: "2.0-1", Architecture: "source", SourceArchitecture: "any", IsSource: true}, - &Package{Name: "lib", Version: "0.9", Architecture: "source", SourceArchitecture: "any", IsSource: true}, + &Package{Name: "postfix", Version: "1.3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}}, + &Package{Name: "app", Version: "1.1~bp1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}}, + &Package{Name: "aa", Version: "2.0-1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}}, + &Package{Name: "lib", Version: "0.9", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}}, } } @@ -302,244 +301,3 @@ func (s *PackageListSuite) TestArchitectures(c *C) { sort.Strings(archs) c.Check(archs, DeepEquals, []string{"amd64", "arm", "i386", "s390"}) } - -func (s *PackageListSuite) TestNewPackageListFromRefList(c *C) { - db, _ := database.OpenDB(c.MkDir()) - coll := NewPackageCollection(db) - coll.Update(s.p1) - coll.Update(s.p3) - - s.list.Add(s.p1) - s.list.Add(s.p3) - s.list.Add(s.p5) - s.list.Add(s.p6) - - reflist := NewPackageRefListFromPackageList(s.list) - - _, err := NewPackageListFromRefList(reflist, coll) - c.Assert(err, ErrorMatches, "unable to load package with key.*") - - coll.Update(s.p5) - coll.Update(s.p6) - - list, err := NewPackageListFromRefList(reflist, coll) - c.Assert(err, IsNil) - c.Check(list.Len(), Equals, 4) - c.Check(list.Add(s.p4), ErrorMatches, "conflict in package.*") - - list, err = NewPackageListFromRefList(nil, coll) - c.Assert(err, IsNil) - c.Check(list.Len(), Equals, 0) -} - -func (s *PackageListSuite) TestNewPackageRefList(c *C) { - s.list.Add(s.p1) - s.list.Add(s.p3) - s.list.Add(s.p5) - s.list.Add(s.p6) - - reflist := NewPackageRefListFromPackageList(s.list) - c.Assert(reflist.Len(), Equals, 4) - c.Check(reflist.Refs[0], DeepEquals, []byte(s.p1.Key())) - c.Check(reflist.Refs[1], DeepEquals, []byte(s.p6.Key())) - c.Check(reflist.Refs[2], DeepEquals, []byte(s.p5.Key())) - c.Check(reflist.Refs[3], DeepEquals, []byte(s.p3.Key())) - - reflist = NewPackageRefList() - c.Check(reflist.Len(), Equals, 0) -} - -func (s *PackageListSuite) TestPackageRefListEncodeDecode(c *C) { - s.list.Add(s.p1) - s.list.Add(s.p3) - s.list.Add(s.p5) - s.list.Add(s.p6) - - reflist := NewPackageRefListFromPackageList(s.list) - - reflist2 := &PackageRefList{} - err := reflist2.Decode(reflist.Encode()) - c.Assert(err, IsNil) - c.Check(reflist2.Len(), Equals, reflist.Len()) - c.Check(reflist2.Refs, DeepEquals, reflist.Refs) -} - -func (s *PackageListSuite) TestPackageRefListForeach(c *C) { - s.list.Add(s.p1) - s.list.Add(s.p3) - s.list.Add(s.p5) - s.list.Add(s.p6) - - reflist := NewPackageRefListFromPackageList(s.list) - - Len := 0 - err := reflist.ForEach(func([]byte) error { - Len++ - return nil - }) - - c.Check(Len, Equals, 4) - c.Check(err, IsNil) - - e := errors.New("b") - - err = reflist.ForEach(func([]byte) error { - return e - }) - - c.Check(err, Equals, e) -} - -func (s *PackageListSuite) TestSubstract(c *C) { - r1 := []byte("r1") - r2 := []byte("r2") - r3 := []byte("r3") - r4 := []byte("r4") - r5 := []byte("r5") - - empty := &PackageRefList{Refs: [][]byte{}} - l1 := &PackageRefList{Refs: [][]byte{r1, r2, r3, r4}} - l2 := &PackageRefList{Refs: [][]byte{r1, r3}} - l3 := &PackageRefList{Refs: [][]byte{r2, r4}} - l4 := &PackageRefList{Refs: [][]byte{r4, r5}} - l5 := &PackageRefList{Refs: [][]byte{r1, r2, r3}} - - c.Check(l1.Substract(empty), DeepEquals, l1) - c.Check(l1.Substract(l2), DeepEquals, l3) - c.Check(l1.Substract(l3), DeepEquals, l2) - c.Check(l1.Substract(l4), DeepEquals, l5) - c.Check(empty.Substract(l1), DeepEquals, empty) - c.Check(l2.Substract(l3), DeepEquals, l2) -} - -func (s *PackageListSuite) TestDiff(c *C) { - db, _ := database.OpenDB(c.MkDir()) - coll := NewPackageCollection(db) - - packages := []*Package{ - &Package{Name: "lib", Version: "1.0", Architecture: "i386"}, //0 - &Package{Name: "dpkg", Version: "1.7", Architecture: "i386"}, //1 - &Package{Name: "data", Version: "1.1~bp1", Architecture: "all"}, //2 - &Package{Name: "app", Version: "1.1~bp1", Architecture: "i386"}, //3 - &Package{Name: "app", Version: "1.1~bp2", Architecture: "i386"}, //4 - &Package{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5 - &Package{Name: "xyz", Version: "3.0", Architecture: "sparc"}, //6 - } - - for _, p := range packages { - coll.Update(p) - } - - listA := NewPackageList() - listA.Add(packages[0]) - listA.Add(packages[1]) - listA.Add(packages[2]) - listA.Add(packages[3]) - listA.Add(packages[6]) - - listB := NewPackageList() - listB.Add(packages[0]) - listB.Add(packages[2]) - listB.Add(packages[4]) - listB.Add(packages[5]) - - reflistA := NewPackageRefListFromPackageList(listA) - reflistB := NewPackageRefListFromPackageList(listB) - - diffAA, err := reflistA.Diff(reflistA, coll) - c.Check(err, IsNil) - c.Check(diffAA, HasLen, 0) - - diffAB, err := reflistA.Diff(reflistB, coll) - c.Check(err, IsNil) - c.Check(diffAB, HasLen, 4) - - c.Check(diffAB[0].Left, IsNil) - c.Check(diffAB[0].Right.String(), Equals, "app_1.1~bp2_amd64") - - c.Check(diffAB[1].Left.String(), Equals, "app_1.1~bp1_i386") - c.Check(diffAB[1].Right.String(), Equals, "app_1.1~bp2_i386") - - c.Check(diffAB[2].Left.String(), Equals, "dpkg_1.7_i386") - c.Check(diffAB[2].Right, IsNil) - - c.Check(diffAB[3].Left.String(), Equals, "xyz_3.0_sparc") - c.Check(diffAB[3].Right, IsNil) - - diffBA, err := reflistB.Diff(reflistA, coll) - c.Check(err, IsNil) - c.Check(diffBA, HasLen, 4) - - c.Check(diffBA[0].Right, IsNil) - c.Check(diffBA[0].Left.String(), Equals, "app_1.1~bp2_amd64") - - c.Check(diffBA[1].Right.String(), Equals, "app_1.1~bp1_i386") - c.Check(diffBA[1].Left.String(), Equals, "app_1.1~bp2_i386") - - c.Check(diffBA[2].Right.String(), Equals, "dpkg_1.7_i386") - c.Check(diffBA[2].Left, IsNil) - - c.Check(diffBA[3].Right.String(), Equals, "xyz_3.0_sparc") - c.Check(diffBA[3].Left, IsNil) - -} - -func (s *PackageListSuite) TestMerge(c *C) { - db, _ := database.OpenDB(c.MkDir()) - coll := NewPackageCollection(db) - - packages := []*Package{ - &Package{Name: "lib", Version: "1.0", Architecture: "i386"}, //0 - &Package{Name: "dpkg", Version: "1.7", Architecture: "i386"}, //1 - &Package{Name: "data", Version: "1.1~bp1", Architecture: "all"}, //2 - &Package{Name: "app", Version: "1.1~bp1", Architecture: "i386"}, //3 - &Package{Name: "app", Version: "1.1~bp2", Architecture: "i386"}, //4 - &Package{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5 - &Package{Name: "dpkg", Version: "1.0", Architecture: "i386"}, //6 - &Package{Name: "xyz", Version: "1.0", Architecture: "sparc"}, //7 - } - - for _, p := range packages { - coll.Update(p) - } - - listA := NewPackageList() - listA.Add(packages[0]) - listA.Add(packages[1]) - listA.Add(packages[2]) - listA.Add(packages[3]) - listA.Add(packages[7]) - - listB := NewPackageList() - listB.Add(packages[0]) - listB.Add(packages[2]) - listB.Add(packages[4]) - listB.Add(packages[5]) - listB.Add(packages[6]) - - reflistA := NewPackageRefListFromPackageList(listA) - reflistB := NewPackageRefListFromPackageList(listB) - - toStrSlice := func(reflist *PackageRefList) (result []string) { - result = make([]string, reflist.Len()) - for i, r := range reflist.Refs { - result[i] = string(r) - } - return - } - - mergeAB := reflistA.Merge(reflistB, true) - mergeBA := reflistB.Merge(reflistA, true) - - c.Check(toStrSlice(mergeAB), DeepEquals, - []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 lib 1.0", "Psparc xyz 1.0"}) - c.Check(toStrSlice(mergeBA), DeepEquals, - []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"}) - - mergeABall := reflistA.Merge(reflistB, false) - mergeBAall := reflistB.Merge(reflistA, false) - - c.Check(mergeABall, DeepEquals, mergeBAall) - c.Check(toStrSlice(mergeBAall), DeepEquals, - []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"}) -} diff --git a/debian/package.go b/debian/package.go index 73b2643e..7d668e3d 100644 --- a/debian/package.go +++ b/debian/package.go @@ -1,54 +1,16 @@ package debian import ( - "bytes" "fmt" "github.com/smira/aptly/aptly" - "github.com/smira/aptly/database" "github.com/smira/aptly/utils" - "github.com/ugorji/go/codec" - "os" "path/filepath" "strconv" "strings" ) -// PackageFile is a single file entry in package -type PackageFile struct { - // Filename is name of file for the package (without directory) - Filename string - // Hashes for the file - Checksums utils.ChecksumInfo - // Temporary field used while downloading, stored relative path on the mirror - downloadPath string -} - -// Verify that package file is present and correct -func (f *PackageFile) Verify(packagePool aptly.PackagePool) (bool, error) { - poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5) - if err != nil { - return false, err - } - - st, err := os.Stat(poolPath) - if err != nil { - return false, nil - } - - // verify size - // TODO: verify checksum if configured - return st.Size() == f.Checksums.Size, nil -} - -// DownloadURL return relative URL to package download location -func (f *PackageFile) DownloadURL() string { - return filepath.Join(f.downloadPath, f.Filename) -} - // Package is single instance of Debian package type Package struct { - // Is this source package - IsSource bool // Basic package properties Name string Version string @@ -58,33 +20,18 @@ type Package struct { SourceArchitecture string // For binary package, name of source package Source string - // Various dependencies - Provides []string - Depends []string - BuildDepends []string - BuildDependsInDep []string - PreDepends []string - Suggests []string - Recommends []string - // Files in package - Files []PackageFile - // Extra information from stanza - Extra Stanza -} - -func parseDependencies(input Stanza, key string) []string { - value, ok := input[key] - if !ok { - return nil - } - - delete(input, key) - - result := strings.Split(value, ",") - for i := range result { - result[i] = strings.TrimSpace(result[i]) - } - return result + // List of virtual packages this package provides + Provides []string + // Is this source package + IsSource bool + // Hash of files section + FilesHash uint64 + // Offload fields + deps *PackageDependencies + extra *Stanza + files *PackageFiles + // Mother collection + collection *PackageCollection } // NewPackageFromControlFile creates Package from parsed Debian control file @@ -94,7 +41,6 @@ func NewPackageFromControlFile(input Stanza) *Package { Version: input["Version"], Architecture: input["Architecture"], Source: input["Source"], - Files: make([]PackageFile, 0, 1), } delete(input, "Package") @@ -104,7 +50,7 @@ func NewPackageFromControlFile(input Stanza) *Package { filesize, _ := strconv.ParseInt(input["Size"], 10, 64) - result.Files = append(result.Files, PackageFile{ + result.UpdateFiles(PackageFiles{PackageFile{ Filename: filepath.Base(input["Filename"]), downloadPath: filepath.Dir(input["Filename"]), Checksums: utils.ChecksumInfo{ @@ -113,7 +59,7 @@ func NewPackageFromControlFile(input Stanza) *Package { SHA1: strings.TrimSpace(input["SHA1"]), SHA256: strings.TrimSpace(input["SHA256"]), }, - }) + }}) delete(input, "Filename") delete(input, "MD5sum") @@ -121,13 +67,16 @@ func NewPackageFromControlFile(input Stanza) *Package { delete(input, "SHA256") delete(input, "Size") - result.Depends = parseDependencies(input, "Depends") - result.PreDepends = parseDependencies(input, "Pre-Depends") - result.Suggests = parseDependencies(input, "Suggests") - result.Recommends = parseDependencies(input, "Recommends") + depends := &PackageDependencies{} + depends.Depends = parseDependencies(input, "Depends") + depends.PreDepends = parseDependencies(input, "Pre-Depends") + depends.Suggests = parseDependencies(input, "Suggests") + depends.Recommends = parseDependencies(input, "Recommends") + result.deps = depends + result.Provides = parseDependencies(input, "Provides") - result.Extra = input + result.extra = &input return result } @@ -146,6 +95,8 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) { delete(input, "Version") delete(input, "Architecture") + files := make(PackageFiles, 0, 3) + parseSums := func(field string, setter func(sum *utils.ChecksumInfo, data string)) error { for _, line := range strings.Split(input[field], "\n") { line = strings.TrimSpace(line) @@ -167,7 +118,7 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) { found := false pos := 0 - for i, file := range result.Files { + for i, file := range files { if file.Filename == filename { found = true pos = i @@ -176,12 +127,12 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) { } if !found { - result.Files = append(result.Files, PackageFile{Filename: filename, downloadPath: input["Directory"]}) - pos = len(result.Files) - 1 + files = append(files, PackageFile{Filename: filename, downloadPath: input["Directory"]}) + pos = len(files) - 1 } - result.Files[pos].Checksums.Size = size - setter(&result.Files[pos].Checksums, parts[0]) + files[pos].Checksums.Size = size + setter(&files[pos].Checksums, parts[0]) } delete(input, field) @@ -202,46 +153,21 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) { return nil, err } - result.BuildDepends = parseDependencies(input, "Build-Depends") - result.BuildDependsInDep = parseDependencies(input, "Build-Depends-Indep") + result.UpdateFiles(files) - result.Extra = input + depends := &PackageDependencies{} + depends.BuildDepends = parseDependencies(input, "Build-Depends") + depends.BuildDependsInDep = parseDependencies(input, "Build-Depends-Indep") + result.deps = depends + + result.extra = &input return result, nil } // Key returns unique key identifying package -func (p *Package) Key() []byte { - return []byte("P" + p.Architecture + " " + p.Name + " " + p.Version) -} - -// Internal buffer reused by all Package.Encode operations -var encodeBuf bytes.Buffer - -// Encode does msgpack encoding of Package, []byte should be copied, as buffer would -// be used for the next call to Encode -func (p *Package) Encode() []byte { - encodeBuf.Reset() - - encoder := codec.NewEncoder(&encodeBuf, &codec.MsgpackHandle{}) - encoder.Encode(p) - - return encodeBuf.Bytes() -} - -// Decode decodes msgpack representation into Package -func (p *Package) Decode(input []byte) error { - decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{}) - err := decoder.Decode(p) - if err != nil { - return err - } - - for i := range p.Files { - p.Files[i].Filename = filepath.Base(p.Files[i].Filename) - } - - return nil +func (p *Package) Key(prefix string) []byte { + return []byte(prefix + "P" + p.Architecture + " " + p.Name + " " + p.Version) } // String creates readable representation @@ -291,21 +217,23 @@ func (p *Package) MatchesDependency(dep Dependency) bool { // GetDependencies compiles list of dependenices by flags from options func (p *Package) GetDependencies(options int) (dependencies []string) { + deps := p.Deps() + dependencies = make([]string, 0, 30) - dependencies = append(dependencies, p.Depends...) - dependencies = append(dependencies, p.PreDepends...) + dependencies = append(dependencies, deps.Depends...) + dependencies = append(dependencies, deps.PreDepends...) if options&DepFollowRecommends == DepFollowRecommends { - dependencies = append(dependencies, p.Recommends...) + dependencies = append(dependencies, deps.Recommends...) } if options&DepFollowSuggests == DepFollowSuggests { - dependencies = append(dependencies, p.Suggests...) + dependencies = append(dependencies, deps.Suggests...) } if options&DepFollowBuild == DepFollowBuild { - dependencies = append(dependencies, p.BuildDepends...) - dependencies = append(dependencies, p.BuildDependsInDep...) + dependencies = append(dependencies, deps.BuildDepends...) + dependencies = append(dependencies, deps.BuildDependsInDep...) } if options&DepFollowSource == DepFollowSource { @@ -323,9 +251,53 @@ func (p *Package) GetDependencies(options int) (dependencies []string) { return } +// Extra returns Stanza of extra fields (it may load it from collection) +func (p *Package) Extra() Stanza { + if p.extra == nil { + if p.collection == nil { + panic("extra == nil && collection == nil") + } + p.extra = p.collection.loadExtra(p) + } + + return *p.extra +} + +// Deps returns parsed package dependencies (it may load it from collection) +func (p *Package) Deps() *PackageDependencies { + if p.deps == nil { + if p.collection == nil { + panic("deps == nil && collection == nil") + } + + p.deps = p.collection.loadDependencies(p) + } + + return p.deps +} + +// Files returns parsed files records (it may load it from collection) +func (p *Package) Files() PackageFiles { + if p.files == nil { + if p.collection == nil { + panic("files == nil && collection == nil") + } + + p.files = p.collection.loadFiles(p) + } + + return *p.files +} + +// UpdateFiles saves new state of files +func (p *Package) UpdateFiles(files PackageFiles) { + p.files = &files + p.FilesHash = files.Hash() +} + // Stanza creates original stanza from package func (p *Package) Stanza() (result Stanza) { - result = p.Extra.Copy() + result = p.Extra().Copy() result["Package"] = p.Name result["Version"] = p.Version @@ -339,7 +311,7 @@ func (p *Package) Stanza() (result Stanza) { if p.IsSource { md5, sha1, sha256 := make([]string, 0), make([]string, 0), make([]string, 0) - for _, f := range p.Files { + for _, f := range p.Files() { if f.Checksums.MD5 != "" { md5 = append(md5, fmt.Sprintf(" %s %d %s\n", f.Checksums.MD5, f.Checksums.Size, f.Filename)) } @@ -355,39 +327,42 @@ func (p *Package) Stanza() (result Stanza) { result["Checksums-Sha1"] = strings.Join(sha1, "") result["Checksums-Sha256"] = strings.Join(sha256, "") } else { - result["Filename"] = p.Files[0].DownloadURL() - if p.Files[0].Checksums.MD5 != "" { - result["MD5sum"] = p.Files[0].Checksums.MD5 + f := p.Files()[0] + result["Filename"] = f.DownloadURL() + if f.Checksums.MD5 != "" { + result["MD5sum"] = f.Checksums.MD5 } - if p.Files[0].Checksums.SHA1 != "" { - result["SHA1"] = " " + p.Files[0].Checksums.SHA1 + if f.Checksums.SHA1 != "" { + result["SHA1"] = " " + f.Checksums.SHA1 } - if p.Files[0].Checksums.SHA256 != "" { - result["SHA256"] = " " + p.Files[0].Checksums.SHA256 + if f.Checksums.SHA256 != "" { + result["SHA256"] = " " + f.Checksums.SHA256 } - result["Size"] = fmt.Sprintf("%d", p.Files[0].Checksums.Size) + result["Size"] = fmt.Sprintf("%d", f.Checksums.Size) } - if p.Depends != nil { - result["Depends"] = strings.Join(p.Depends, ", ") + deps := p.Deps() + + if deps.Depends != nil { + result["Depends"] = strings.Join(deps.Depends, ", ") } - if p.PreDepends != nil { - result["Pre-Depends"] = strings.Join(p.PreDepends, ", ") + if deps.PreDepends != nil { + result["Pre-Depends"] = strings.Join(deps.PreDepends, ", ") } - if p.Suggests != nil { - result["Suggests"] = strings.Join(p.Suggests, ", ") + if deps.Suggests != nil { + result["Suggests"] = strings.Join(deps.Suggests, ", ") } - if p.Recommends != nil { - result["Recommends"] = strings.Join(p.Recommends, ", ") + if deps.Recommends != nil { + result["Recommends"] = strings.Join(deps.Recommends, ", ") } if p.Provides != nil { result["Provides"] = strings.Join(p.Provides, ", ") } - if p.BuildDepends != nil { - result["Build-Depends"] = strings.Join(p.BuildDepends, ", ") + if deps.BuildDepends != nil { + result["Build-Depends"] = strings.Join(deps.BuildDepends, ", ") } - if p.BuildDependsInDep != nil { - result["Build-Depends-Indep"] = strings.Join(p.BuildDependsInDep, ", ") + if deps.BuildDependsInDep != nil { + result["Build-Depends-Indep"] = strings.Join(deps.BuildDependsInDep, ", ") } return @@ -395,33 +370,9 @@ func (p *Package) Stanza() (result Stanza) { // Equals compares two packages to be identical func (p *Package) Equals(p2 *Package) bool { - if len(p.Files) != len(p2.Files) { - return false - } - - for _, f := range p.Files { - found := false - for _, f2 := range p2.Files { - if f2.Filename == f.Filename { - found = true - if f.Checksums != f2.Checksums { - return false - } - break - } - - } - if !found { - return false - } - } - return p.Name == p2.Name && p.Version == p2.Version && p.SourceArchitecture == p2.SourceArchitecture && - p.Architecture == p2.Architecture && utils.StrSlicesEqual(p.Depends, p2.Depends) && - utils.StrSlicesEqual(p.PreDepends, p2.PreDepends) && utils.StrSlicesEqual(p.Suggests, p2.Suggests) && - utils.StrSlicesEqual(p.Recommends, p2.Recommends) && utils.StrMapsEqual(p.Extra, p2.Extra) && - p.Source == p2.Source && utils.StrSlicesEqual(p.Provides, p2.Provides) && utils.StrSlicesEqual(p.BuildDepends, p2.BuildDepends) && - utils.StrSlicesEqual(p.BuildDependsInDep, p2.BuildDependsInDep) && p.IsSource == p2.IsSource + p.Architecture == p2.Architecture && p.Source == p2.Source && p.IsSource == p2.IsSource && + p.FilesHash == p2.FilesHash } // LinkFromPool links package file from pool to dist's pool location @@ -431,7 +382,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP return err } - for i, f := range p.Files { + for i, f := range p.Files() { sourcePath, err := packagePool.Path(f.Filename, f.Checksums.MD5) if err != nil { return err @@ -444,9 +395,9 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP dir := filepath.Dir(relPath) if p.IsSource { - p.Extra["Directory"] = dir + p.Extra()["Directory"] = dir } else { - p.Files[i].downloadPath = dir + p.Files()[i].downloadPath = dir } } @@ -487,7 +438,7 @@ type PackageDownloadTask struct { func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageDownloadTask, err error) { result = make([]PackageDownloadTask, 0, 1) - for _, f := range p.Files { + for _, f := range p.Files() { poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5) if err != nil { return nil, err @@ -510,7 +461,7 @@ func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageD func (p *Package) VerifyFiles(packagePool aptly.PackagePool) (result bool, err error) { result = true - for _, f := range p.Files { + for _, f := range p.Files() { result, err = f.Verify(packagePool) if err != nil || !result { return @@ -523,9 +474,9 @@ func (p *Package) VerifyFiles(packagePool aptly.PackagePool) (result bool, err e // FilepathList returns list of paths to files in package repository func (p *Package) FilepathList(packagePool aptly.PackagePool) ([]string, error) { var err error - result := make([]string, len(p.Files)) + result := make([]string, len(p.Files())) - for i, f := range p.Files { + for i, f := range p.Files() { result[i], err = packagePool.RelativePath(f.Filename, f.Checksums.MD5) if err != nil { return nil, err @@ -534,83 +485,3 @@ func (p *Package) FilepathList(packagePool aptly.PackagePool) ([]string, error) return result, nil } - -// PackageCollection does management of packages in DB -type PackageCollection struct { - db database.Storage -} - -// NewPackageCollection creates new PackageCollection and binds it to database -func NewPackageCollection(db database.Storage) *PackageCollection { - return &PackageCollection{ - db: db, - } -} - -// ByKey find package in DB by its key -func (collection *PackageCollection) ByKey(key []byte) (*Package, error) { - encoded, err := collection.db.Get(key) - if err != nil { - return nil, err - } - - p := &Package{} - err = p.Decode(encoded) - if err != nil { - return nil, err - } - return p, nil -} - -// Update adds or updates information about package in DB -func (collection *PackageCollection) Update(p *Package) error { - existing, err := collection.ByKey(p.Key()) - if err == nil { - // check for conflict - if existing.Equals(p) { - // packages are the same, no need to update - return nil - } - - // if .Files is different, consider to be conflict - if len(p.Files) != len(existing.Files) { - return fmt.Errorf("unable to save: %s, conflict with existing packge", p) - } - - for _, f := range p.Files { - found := false - for _, f2 := range existing.Files { - if f2.Filename == f.Filename { - found = true - if f.Checksums != f2.Checksums { - return fmt.Errorf("unable to save: %s, conflict with existing packge", p) - } - break - } - - } - if !found { - return fmt.Errorf("unable to save: %s, conflict with existing packge", p) - } - } - - // ok, .Files are the same, but some meta-data is different, proceed to saving - } else { - if err != database.ErrNotFound { - return err - } - // ok, package doesn't exist yet - } - - return collection.db.Put(p.Key(), p.Encode()) -} - -// AllPackageRefs returns list of all packages as PackageRefList -func (collection *PackageCollection) AllPackageRefs() *PackageRefList { - return &PackageRefList{Refs: collection.db.KeysByPrefix([]byte("P"))} -} - -// DeleteByKey deletes package in DB by key -func (collection *PackageCollection) DeleteByKey(key []byte) error { - return collection.db.Delete(key) -} diff --git a/debian/package_collection.go b/debian/package_collection.go new file mode 100644 index 00000000..4454fde6 --- /dev/null +++ b/debian/package_collection.go @@ -0,0 +1,256 @@ +package debian + +import ( + "bytes" + "fmt" + "github.com/smira/aptly/database" + "github.com/ugorji/go/codec" + "path/filepath" +) + +// PackageCollection does management of packages in DB +type PackageCollection struct { + db database.Storage + encodeBuffer bytes.Buffer +} + +// NewPackageCollection creates new PackageCollection and binds it to database +func NewPackageCollection(db database.Storage) *PackageCollection { + return &PackageCollection{ + db: db, + } +} + +// oldPackage is Package struct for aptly < 0.4 with all fields in one struct +// It is used to decode old aptly DBs +type oldPackage struct { + IsSource bool + Name string + Version string + Architecture string + SourceArchitecture string + Source string + Provides []string + Depends []string + BuildDepends []string + BuildDependsInDep []string + PreDepends []string + Suggests []string + Recommends []string + Files []PackageFile + Extra Stanza +} + +// ByKey find package in DB by its key +func (collection *PackageCollection) ByKey(key []byte) (*Package, error) { + encoded, err := collection.db.Get(key) + if err != nil { + return nil, err + } + + p := &Package{} + + if len(encoded) > 2 && (encoded[0] != 0xc1 || encoded[1] != 0x1) { + oldp := &oldPackage{} + + decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{}) + err = decoder.Decode(oldp) + if err != nil { + return nil, err + } + + p.Name = oldp.Name + p.Version = oldp.Version + p.Architecture = oldp.Architecture + p.IsSource = oldp.IsSource + p.SourceArchitecture = oldp.SourceArchitecture + p.Source = oldp.Source + p.Provides = oldp.Provides + + p.deps = &PackageDependencies{ + Depends: oldp.Depends, + BuildDepends: oldp.BuildDepends, + BuildDependsInDep: oldp.BuildDependsInDep, + PreDepends: oldp.PreDepends, + Suggests: oldp.Suggests, + Recommends: oldp.Recommends, + } + + p.extra = &oldp.Extra + for i := range oldp.Files { + oldp.Files[i].Filename = filepath.Base(oldp.Files[i].Filename) + } + p.UpdateFiles(PackageFiles(oldp.Files)) + + // Save in new format + err = collection.internalUpdate(p) + if err != nil { + return nil, err + } + } else { + decoder := codec.NewDecoderBytes(encoded[2:], &codec.MsgpackHandle{}) + err = decoder.Decode(p) + if err != nil { + return nil, err + } + } + + p.collection = collection + + return p, nil +} + +// loadExtra loads Stanza with all the xtra information about the package +func (collection *PackageCollection) loadExtra(p *Package) *Stanza { + encoded, err := collection.db.Get(p.Key("xE")) + if err != nil { + panic("unable to load extra") + } + + stanza := &Stanza{} + + decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{}) + err = decoder.Decode(stanza) + if err != nil { + panic("unable to decode extra") + } + + return stanza +} + +// loadDependencies loads dependencies for the package +func (collection *PackageCollection) loadDependencies(p *Package) *PackageDependencies { + encoded, err := collection.db.Get(p.Key("xD")) + if err != nil { + panic(fmt.Sprintf("unable to load deps: %s, %s", p, err)) + } + + deps := &PackageDependencies{} + + decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{}) + err = decoder.Decode(deps) + if err != nil { + panic("unable to decode deps") + } + + return deps +} + +// loadFiles loads additional PackageFiles record +func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles { + encoded, err := collection.db.Get(p.Key("xF")) + if err != nil { + panic("unable to load files") + } + + files := &PackageFiles{} + + decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{}) + err = decoder.Decode(files) + if err != nil { + panic("unable to decode files") + } + + return files +} + +// Update adds or updates information about package in DB checking for conficts first +func (collection *PackageCollection) Update(p *Package) error { + existing, err := collection.ByKey(p.Key("")) + if err == nil { + // if .Files is different, consider to be conflict + if p.FilesHash != existing.FilesHash { + return fmt.Errorf("unable to save: %s, conflict with existing packge", p) + } + // ok, .Files are the same, but maybe some meta-data is different, proceed to saving + } else { + if err != database.ErrNotFound { + return err + } + // ok, package doesn't exist yet + } + + return collection.internalUpdate(p) +} + +// internalUpdate updates information in DB about package and offloaded fields +func (collection *PackageCollection) internalUpdate(p *Package) error { + encoder := codec.NewEncoder(&collection.encodeBuffer, &codec.MsgpackHandle{}) + + collection.encodeBuffer.Reset() + collection.encodeBuffer.WriteByte(0xc1) + collection.encodeBuffer.WriteByte(0x1) + err := encoder.Encode(p) + if err != nil { + return err + } + + err = collection.db.Put(p.Key(""), collection.encodeBuffer.Bytes()) + if err != nil { + return err + } + + // Encode offloaded fields one by one + if p.files != nil { + collection.encodeBuffer.Reset() + err = encoder.Encode(*p.files) + if err != nil { + return err + } + + err = collection.db.Put(p.Key("xF"), collection.encodeBuffer.Bytes()) + if err != nil { + return err + } + } + + if p.deps != nil { + collection.encodeBuffer.Reset() + err = encoder.Encode(*p.deps) + if err != nil { + return err + } + + err = collection.db.Put(p.Key("xD"), collection.encodeBuffer.Bytes()) + if err != nil { + return err + } + + p.deps = nil + } + + if p.extra != nil { + collection.encodeBuffer.Reset() + err = encoder.Encode(*p.extra) + if err != nil { + return err + } + + err = collection.db.Put(p.Key("xE"), collection.encodeBuffer.Bytes()) + if err != nil { + return err + } + + p.extra = nil + } + + p.collection = collection + + return nil +} + +// AllPackageRefs returns list of all packages as PackageRefList +func (collection *PackageCollection) AllPackageRefs() *PackageRefList { + return &PackageRefList{Refs: collection.db.KeysByPrefix([]byte("P"))} +} + +// DeleteByKey deletes package in DB by key +func (collection *PackageCollection) DeleteByKey(key []byte) error { + for _, key := range [][]byte{key, append([]byte("xF"), key...), append([]byte("xD"), key...), append([]byte("xE"), key...)} { + err := collection.db.Delete(key) + if err != nil { + return err + } + } + return nil +} diff --git a/debian/package_collection_test.go b/debian/package_collection_test.go new file mode 100644 index 00000000..46032654 --- /dev/null +++ b/debian/package_collection_test.go @@ -0,0 +1,93 @@ +package debian + +import ( + "github.com/smira/aptly/database" + . "launchpad.net/gocheck" +) + +type PackageCollectionSuite struct { + collection *PackageCollection + p *Package + db database.Storage +} + +var _ = Suite(&PackageCollectionSuite{}) + +func (s *PackageCollectionSuite) SetUpTest(c *C) { + s.p = NewPackageFromControlFile(packageStanza.Copy()) + s.db, _ = database.OpenDB(c.MkDir()) + s.collection = NewPackageCollection(s.db) +} + +func (s *PackageCollectionSuite) TearDownTest(c *C) { + s.db.Close() +} + +func (s *PackageCollectionSuite) TestUpdate(c *C) { + // package doesn't exist, update ok + err := s.collection.Update(s.p) + c.Assert(err, IsNil) + res, err := s.collection.ByKey(s.p.Key("")) + c.Assert(err, IsNil) + c.Assert(res.Equals(s.p), Equals, true) + + // same package, ok + p2 := NewPackageFromControlFile(packageStanza.Copy()) + err = s.collection.Update(p2) + c.Assert(err, IsNil) + res, err = s.collection.ByKey(p2.Key("")) + c.Assert(err, IsNil) + c.Assert(res.Equals(s.p), Equals, true) + + // change some metadata + p2.Source = "lala" + err = s.collection.Update(p2) + c.Assert(err, IsNil) + res, err = s.collection.ByKey(p2.Key("")) + c.Assert(err, IsNil) + c.Assert(res.Equals(s.p), Equals, false) + c.Assert(res.Equals(p2), Equals, true) + + // change file info + p2 = NewPackageFromControlFile(packageStanza.Copy()) + p2.UpdateFiles(nil) + res, err = s.collection.ByKey(p2.Key("")) + err = s.collection.Update(p2) + c.Assert(err, ErrorMatches, ".*conflict with existing packge") + p2 = NewPackageFromControlFile(packageStanza.Copy()) + files := p2.Files() + files[0].Checksums.MD5 = "abcdef" + p2.UpdateFiles(files) + res, err = s.collection.ByKey(p2.Key("")) + err = s.collection.Update(p2) + c.Assert(err, ErrorMatches, ".*conflict with existing packge") +} + +func (s *PackageCollectionSuite) TestByKey(c *C) { + err := s.collection.Update(s.p) + c.Assert(err, IsNil) + + p2, err := s.collection.ByKey(s.p.Key("")) + c.Assert(err, IsNil) + c.Assert(p2.Equals(s.p), Equals, true) +} + +func (s *PackageCollectionSuite) TestAllPackageRefs(c *C) { + err := s.collection.Update(s.p) + c.Assert(err, IsNil) + + refs := s.collection.AllPackageRefs() + c.Check(refs.Len(), Equals, 1) + c.Check(refs.Refs[0], DeepEquals, s.p.Key("")) +} + +func (s *PackageCollectionSuite) TestDeleteByKey(c *C) { + err := s.collection.Update(s.p) + c.Assert(err, IsNil) + + err = s.collection.DeleteByKey(s.p.Key("")) + c.Check(err, IsNil) + + _, err = s.collection.ByKey(s.p.Key("")) + c.Check(err, ErrorMatches, "key not found") +} diff --git a/debian/package_deps.go b/debian/package_deps.go new file mode 100644 index 00000000..0de14925 --- /dev/null +++ b/debian/package_deps.go @@ -0,0 +1,30 @@ +package debian + +import ( + "strings" +) + +// PackageDependencies are various parsed dependencies +type PackageDependencies struct { + Depends []string + BuildDepends []string + BuildDependsInDep []string + PreDepends []string + Suggests []string + Recommends []string +} + +func parseDependencies(input Stanza, key string) []string { + value, ok := input[key] + if !ok { + return nil + } + + delete(input, key) + + result := strings.Split(value, ",") + for i := range result { + result[i] = strings.TrimSpace(result[i]) + } + return result +} diff --git a/debian/package_files.go b/debian/package_files.go new file mode 100644 index 00000000..236b5bfe --- /dev/null +++ b/debian/package_files.go @@ -0,0 +1,78 @@ +package debian + +import ( + "encoding/binary" + "github.com/smira/aptly/aptly" + "github.com/smira/aptly/utils" + "hash/fnv" + "os" + "path/filepath" + "sort" +) + +// PackageFile is a single file entry in package +type PackageFile struct { + // Filename is name of file for the package (without directory) + Filename string + // Hashes for the file + Checksums utils.ChecksumInfo + // Temporary field used while downloading, stored relative path on the mirror + downloadPath string +} + +// Verify that package file is present and correct +func (f *PackageFile) Verify(packagePool aptly.PackagePool) (bool, error) { + poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5) + if err != nil { + return false, err + } + + st, err := os.Stat(poolPath) + if err != nil { + return false, nil + } + + // verify size + // TODO: verify checksum if configured + return st.Size() == f.Checksums.Size, nil +} + +// DownloadURL return relative URL to package download location +func (f *PackageFile) DownloadURL() string { + return filepath.Join(f.downloadPath, f.Filename) +} + +// PackageFiles is collection of package files +type PackageFiles []PackageFile + +// Hash compute hash of all file items, sorting them first +func (files PackageFiles) Hash() uint64 { + sort.Sort(files) + + h := fnv.New64a() + + for _, f := range files { + h.Write([]byte(f.Filename)) + binary.Write(h, binary.BigEndian, f.Checksums.Size) + h.Write([]byte(f.Checksums.MD5)) + h.Write([]byte(f.Checksums.SHA1)) + h.Write([]byte(f.Checksums.SHA256)) + } + + return h.Sum64() +} + +// Len returns number of files +func (files PackageFiles) Len() int { + return len(files) +} + +// Swap swaps elements +func (files PackageFiles) Swap(i, j int) { + files[i], files[j] = files[j], files[i] +} + +// Less compares by filename +func (files PackageFiles) Less(i, j int) bool { + return files[i].Filename < files[j].Filename +} diff --git a/debian/package_files_test.go b/debian/package_files_test.go new file mode 100644 index 00000000..cba5444e --- /dev/null +++ b/debian/package_files_test.go @@ -0,0 +1,61 @@ +package debian + +import ( + "github.com/smira/aptly/files" + "github.com/smira/aptly/utils" + . "launchpad.net/gocheck" + "os" + "path/filepath" +) + +type PackageFilesSuite struct { + files PackageFiles +} + +var _ = Suite(&PackageFilesSuite{}) + +func (s *PackageFilesSuite) SetUpTest(c *C) { + s.files = PackageFiles{PackageFile{ + Filename: "alien-arena-common_7.40-2_i386.deb", + downloadPath: "pool/contrib/a/alien-arena", + Checksums: utils.ChecksumInfo{ + Size: 187518, + MD5: "1e8cba92c41420aa7baa8a5718d67122", + SHA1: "46955e48cad27410a83740a21d766ce362364024", + SHA256: "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", + }}} +} + +func (s *PackageFilesSuite) TestVerify(c *C) { + packagePool := files.NewPackagePool(c.MkDir()) + poolPath, _ := packagePool.Path(s.files[0].Filename, s.files[0].Checksums.MD5) + + result, err := s.files[0].Verify(packagePool) + c.Check(err, IsNil) + c.Check(result, Equals, false) + + err = os.MkdirAll(filepath.Dir(poolPath), 0755) + c.Assert(err, IsNil) + + file, err := os.Create(poolPath) + c.Assert(err, IsNil) + file.WriteString("abcde") + file.Close() + + result, err = s.files[0].Verify(packagePool) + c.Check(err, IsNil) + c.Check(result, Equals, false) + + s.files[0].Checksums.Size = 5 + result, err = s.files[0].Verify(packagePool) + c.Check(err, IsNil) + c.Check(result, Equals, true) +} + +func (s *PackageFilesSuite) TestDownloadURL(c *C) { + c.Check(s.files[0].DownloadURL(), Equals, "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb") +} + +func (s *PackageFilesSuite) TestHash(c *C) { + c.Check(s.files.Hash(), Equals, uint64(0xc8901eedd79ac51b)) +} diff --git a/debian/package_test.go b/debian/package_test.go index e2474d17..9888cb14 100644 --- a/debian/package_test.go +++ b/debian/package_test.go @@ -2,7 +2,6 @@ package debian import ( "bytes" - "github.com/smira/aptly/database" "github.com/smira/aptly/files" "github.com/smira/aptly/utils" . "launchpad.net/gocheck" @@ -24,49 +23,6 @@ func (s *PackageSuite) SetUpTest(c *C) { s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza() } -func (s *PackageSuite) TestPackageFileVerify(c *C) { - packagePool := files.NewPackagePool(c.MkDir()) - p := NewPackageFromControlFile(s.stanza) - poolPath, _ := packagePool.Path(p.Files[0].Filename, p.Files[0].Checksums.MD5) - - result, err := p.Files[0].Verify(packagePool) - c.Check(err, IsNil) - c.Check(result, Equals, false) - - err = os.MkdirAll(filepath.Dir(poolPath), 0755) - c.Assert(err, IsNil) - - file, err := os.Create(poolPath) - c.Assert(err, IsNil) - file.WriteString("abcde") - file.Close() - - result, err = p.Files[0].Verify(packagePool) - c.Check(err, IsNil) - c.Check(result, Equals, false) - - result, err = p.VerifyFiles(packagePool) - c.Check(err, IsNil) - c.Check(result, Equals, false) - - p.Files[0].Checksums.Size = 5 - result, err = p.Files[0].Verify(packagePool) - c.Check(err, IsNil) - c.Check(result, Equals, true) - - result, err = p.VerifyFiles(packagePool) - c.Check(err, IsNil) - c.Check(result, Equals, true) -} - -func (s *PackageSuite) TestPackageFileDownloadURL(c *C) { - p := NewPackageFromControlFile(s.stanza) - - c.Check(p.Files[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb") - c.Check(p.Files[0].downloadPath, Equals, "pool/contrib/a/alien-arena") - c.Check(p.Files[0].DownloadURL(), Equals, "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb") -} - func (s *PackageSuite) TestNewFromPara(c *C) { p := NewPackageFromControlFile(s.stanza) @@ -75,12 +31,12 @@ func (s *PackageSuite) TestNewFromPara(c *C) { c.Check(p.Version, Equals, "7.40-2") c.Check(p.Architecture, Equals, "i386") c.Check(p.Provides, DeepEquals, []string(nil)) - c.Check(p.Files, HasLen, 1) - c.Check(p.Files[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb") - c.Check(p.Files[0].downloadPath, Equals, "pool/contrib/a/alien-arena") - c.Check(p.Files[0].Checksums.Size, Equals, int64(187518)) - c.Check(p.Files[0].Checksums.MD5, Equals, "1e8cba92c41420aa7baa8a5718d67122") - c.Check(p.Depends, DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)"}) + c.Check(p.Files(), HasLen, 1) + c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb") + c.Check(p.Files()[0].downloadPath, Equals, "pool/contrib/a/alien-arena") + c.Check(p.Files()[0].Checksums.Size, Equals, int64(187518)) + c.Check(p.Files()[0].Checksums.MD5, Equals, "1e8cba92c41420aa7baa8a5718d67122") + c.Check(p.deps.Depends, DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)"}) } func (s *PackageSuite) TestNewSourceFromPara(c *C) { @@ -93,28 +49,28 @@ func (s *PackageSuite) TestNewSourceFromPara(c *C) { c.Check(p.Architecture, Equals, "source") c.Check(p.SourceArchitecture, Equals, "all") c.Check(p.Provides, IsNil) - c.Check(p.BuildDepends, DeepEquals, []string{"cdbs", "debhelper (>= 7)", "default-jdk", "maven-debian-helper"}) - c.Check(p.BuildDependsInDep, DeepEquals, []string{"default-jdk-doc", "junit (>= 3.8.1)", "libannotation-indexer-java (>= 1.3)", "libannotation-indexer-java-doc", "libasm3-java", "libmaven-install-plugin-java", "libmaven-javadoc-plugin-java", "libmaven-scm-java", "libmaven2-core-java", "libmaven2-core-java-doc", "libmetainf-services-java", "libmetainf-services-java-doc", "libmaven-plugin-tools-java (>= 2.8)"}) - c.Check(p.Files, HasLen, 3) + c.Check(p.deps.BuildDepends, DeepEquals, []string{"cdbs", "debhelper (>= 7)", "default-jdk", "maven-debian-helper"}) + c.Check(p.deps.BuildDependsInDep, DeepEquals, []string{"default-jdk-doc", "junit (>= 3.8.1)", "libannotation-indexer-java (>= 1.3)", "libannotation-indexer-java-doc", "libasm3-java", "libmaven-install-plugin-java", "libmaven-javadoc-plugin-java", "libmaven-scm-java", "libmaven2-core-java", "libmaven2-core-java-doc", "libmetainf-services-java", "libmetainf-services-java-doc", "libmaven-plugin-tools-java (>= 2.8)"}) + c.Check(p.Files(), HasLen, 3) - c.Check(p.Files[0].Filename, Equals, "access-modifier-checker_1.0-4.dsc") - c.Check(p.Files[0].downloadPath, Equals, "pool/main/a/access-modifier-checker") - c.Check(p.Files[0].Checksums.Size, Equals, int64(3)) - c.Check(p.Files[0].Checksums.MD5, Equals, "900150983cd24fb0d6963f7d28e17f72") - c.Check(p.Files[0].Checksums.SHA1, Equals, "a9993e364706816aba3e25717850c26c9cd0d89d") - c.Check(p.Files[0].Checksums.SHA256, Equals, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + c.Check(p.Files()[0].Filename, Equals, "access-modifier-checker_1.0-4.debian.tar.gz") + c.Check(p.Files()[0].downloadPath, Equals, "pool/main/a/access-modifier-checker") - c.Check(p.Files[1].Filename, Equals, "access-modifier-checker_1.0.orig.tar.gz") - c.Check(p.Files[0].downloadPath, Equals, "pool/main/a/access-modifier-checker") - c.Check(p.Files[1].Checksums.Size, Equals, int64(4)) - c.Check(p.Files[1].Checksums.MD5, Equals, "e2fc714c4727ee9395f324cd2e7f331f") - c.Check(p.Files[1].Checksums.SHA1, Equals, "81fe8bfe87576c3ecb22426f8e57847382917acf") - c.Check(p.Files[1].Checksums.SHA256, Equals, "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589") + c.Check(p.Files()[1].Filename, Equals, "access-modifier-checker_1.0-4.dsc") + c.Check(p.Files()[1].downloadPath, Equals, "pool/main/a/access-modifier-checker") + c.Check(p.Files()[1].Checksums.Size, Equals, int64(3)) + c.Check(p.Files()[1].Checksums.MD5, Equals, "900150983cd24fb0d6963f7d28e17f72") + c.Check(p.Files()[1].Checksums.SHA1, Equals, "a9993e364706816aba3e25717850c26c9cd0d89d") + c.Check(p.Files()[1].Checksums.SHA256, Equals, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") - c.Check(p.Files[2].Filename, Equals, "access-modifier-checker_1.0-4.debian.tar.gz") - c.Check(p.Files[0].downloadPath, Equals, "pool/main/a/access-modifier-checker") + c.Check(p.Files()[2].Filename, Equals, "access-modifier-checker_1.0.orig.tar.gz") + c.Check(p.Files()[2].downloadPath, Equals, "pool/main/a/access-modifier-checker") + c.Check(p.Files()[2].Checksums.Size, Equals, int64(4)) + c.Check(p.Files()[2].Checksums.MD5, Equals, "e2fc714c4727ee9395f324cd2e7f331f") + c.Check(p.Files()[2].Checksums.SHA1, Equals, "81fe8bfe87576c3ecb22426f8e57847382917acf") + c.Check(p.Files()[2].Checksums.SHA256, Equals, "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589") - c.Check(p.Depends, IsNil) + c.Check(p.deps.Depends, IsNil) } func (s *PackageSuite) TestWithProvides(c *C) { @@ -131,21 +87,8 @@ func (s *PackageSuite) TestWithProvides(c *C) { func (s *PackageSuite) TestKey(c *C) { p := NewPackageFromControlFile(s.stanza) - c.Check(p.Key(), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2")) -} - -func (s *PackageSuite) TestEncodeDecode(c *C) { - p := NewPackageFromControlFile(s.stanza) - - // downloadPath would be lost in encode/decode cycle, that's OK - p.Files[0].downloadPath = "" - - encoded := p.Encode() - p2 := &Package{} - err := p2.Decode(encoded) - - c.Assert(err, IsNil) - c.Assert(p2, DeepEquals, p) + c.Check(p.Key(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2")) + c.Check(p.Key("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2")) } func (s *PackageSuite) TestStanza(c *C) { @@ -171,11 +114,13 @@ func (s *PackageSuite) TestEquals(c *C) { p2 := NewPackageFromControlFile(packageStanza.Copy()) c.Check(p.Equals(p2), Equals, true) - p2.Depends = []string{"package1"} - c.Check(p.Equals(p2), Equals, false) + p2.deps.Depends = []string{"package1"} + c.Check(p.Equals(p2), Equals, true) // strange, but Equals doesn't check deep p2 = NewPackageFromControlFile(packageStanza.Copy()) - p2.Files[0].Checksums.MD5 = "abcdefabcdef" + files := p2.Files() + files[0].Checksums.MD5 = "abcdefabcdef" + p2.UpdateFiles(files) c.Check(p.Equals(p2), Equals, false) so, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy()) @@ -183,14 +128,15 @@ func (s *PackageSuite) TestEquals(c *C) { c.Check(so.Equals(so2), Equals, true) - so2.Files[2], so2.Files[1] = so2.Files[1], so2.Files[2] - c.Check(so.Equals(so2), Equals, true) - - so2.Files[2].Checksums.MD5 = "abcde" + files = so2.Files() + files[2].Checksums.MD5 = "abcde" + so2.UpdateFiles(files) c.Check(so.Equals(so2), Equals, false) so2, _ = NewSourcePackageFromControlFile(s.sourceStanza.Copy()) - so2.Files[1].Filename = "other.deb" + files = so2.Files() + files[1].Filename = "other.deb" + so2.UpdateFiles(files) c.Check(so.Equals(so2), Equals, false) } @@ -295,7 +241,7 @@ func (s *PackageSuite) TestLinkFromPool(c *C) { publishedStorage := files.NewPublishedStorage(c.MkDir()) p := NewPackageFromControlFile(s.stanza) - poolPath, _ := packagePool.Path(p.Files[0].Filename, p.Files[0].Checksums.MD5) + poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5) err := os.MkdirAll(filepath.Dir(poolPath), 0755) c.Assert(err, IsNil) @@ -305,13 +251,13 @@ func (s *PackageSuite) TestLinkFromPool(c *C) { err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free") c.Check(err, IsNil) - c.Check(p.Files[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb") - c.Check(p.Files[0].downloadPath, Equals, "pool/non-free/a/alien-arena") + c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb") + c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena") p.IsSource = true err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free") c.Check(err, IsNil) - c.Check(p.Extra["Directory"], Equals, "pool/non-free/a/alien-arena") + c.Check(p.Extra()["Directory"], Equals, "pool/non-free/a/alien-arena") } func (s *PackageSuite) TestFilepathList(c *C) { @@ -326,8 +272,8 @@ func (s *PackageSuite) TestFilepathList(c *C) { func (s *PackageSuite) TestDownloadList(c *C) { packagePool := files.NewPackagePool(c.MkDir()) p := NewPackageFromControlFile(s.stanza) - p.Files[0].Checksums.Size = 5 - poolPath, _ := packagePool.Path(p.Files[0].Filename, p.Files[0].Checksums.MD5) + p.Files()[0].Checksums.Size = 5 + poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5) list, err := p.DownloadList(packagePool) c.Check(err, IsNil) @@ -353,89 +299,29 @@ func (s *PackageSuite) TestDownloadList(c *C) { c.Check(list, DeepEquals, []PackageDownloadTask{}) } -type PackageCollectionSuite struct { - collection *PackageCollection - p *Package - db database.Storage -} +func (s *PackageSuite) TestVerifyFiles(c *C) { + p := NewPackageFromControlFile(s.stanza) -var _ = Suite(&PackageCollectionSuite{}) + packagePool := files.NewPackagePool(c.MkDir()) + poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5) -func (s *PackageCollectionSuite) SetUpTest(c *C) { - s.p = NewPackageFromControlFile(packageStanza.Copy()) - s.db, _ = database.OpenDB(c.MkDir()) - s.collection = NewPackageCollection(s.db) -} - -func (s *PackageCollectionSuite) TearDownTest(c *C) { - s.db.Close() -} - -func (s *PackageCollectionSuite) TestUpdate(c *C) { - // package doesn't exist, update ok - err := s.collection.Update(s.p) - c.Assert(err, IsNil) - res, err := s.collection.ByKey(s.p.Key()) - c.Assert(err, IsNil) - c.Assert(res.Equals(s.p), Equals, true) - - // same package, ok - p2 := NewPackageFromControlFile(packageStanza.Copy()) - err = s.collection.Update(p2) - c.Assert(err, IsNil) - res, err = s.collection.ByKey(p2.Key()) - c.Assert(err, IsNil) - c.Assert(res.Equals(s.p), Equals, true) - - // change some metadata - p2.Source = "lala" - err = s.collection.Update(p2) - c.Assert(err, IsNil) - res, err = s.collection.ByKey(p2.Key()) - c.Assert(err, IsNil) - c.Assert(res.Equals(s.p), Equals, false) - c.Assert(res.Equals(p2), Equals, true) - - // change file info - p2 = NewPackageFromControlFile(packageStanza.Copy()) - p2.Files = nil - res, err = s.collection.ByKey(p2.Key()) - err = s.collection.Update(p2) - c.Assert(err, ErrorMatches, ".*conflict with existing packge") - p2 = NewPackageFromControlFile(packageStanza.Copy()) - p2.Files[0].Checksums.MD5 = "abcdef" - res, err = s.collection.ByKey(p2.Key()) - err = s.collection.Update(p2) - c.Assert(err, ErrorMatches, ".*conflict with existing packge") -} - -func (s *PackageCollectionSuite) TestByKey(c *C) { - err := s.collection.Update(s.p) + err := os.MkdirAll(filepath.Dir(poolPath), 0755) c.Assert(err, IsNil) - p2, err := s.collection.ByKey(s.p.Key()) + file, err := os.Create(poolPath) c.Assert(err, IsNil) - c.Assert(p2.Equals(s.p), Equals, true) -} + file.WriteString("abcde") + file.Close() -func (s *PackageCollectionSuite) TestAllPackageRefs(c *C) { - err := s.collection.Update(s.p) - c.Assert(err, IsNil) - - refs := s.collection.AllPackageRefs() - c.Check(refs.Len(), Equals, 1) - c.Check(refs.Refs[0], DeepEquals, s.p.Key()) -} - -func (s *PackageCollectionSuite) TestDeleteByKey(c *C) { - err := s.collection.Update(s.p) - c.Assert(err, IsNil) - - err = s.collection.DeleteByKey(s.p.Key()) + result, err := p.VerifyFiles(packagePool) c.Check(err, IsNil) + c.Check(result, Equals, false) - _, err = s.collection.ByKey(s.p.Key()) - c.Check(err, ErrorMatches, "key not found") + p.Files()[0].Checksums.Size = 5 + + result, err = p.VerifyFiles(packagePool) + c.Check(err, IsNil) + c.Check(result, Equals, true) } var packageStanza = Stanza{"Source": "alien-arena", "Pre-Depends": "dpkg (>= 1.6)", "Suggests": "alien-arena-mars", "Recommends": "aliean-arena-luna", "Depends": "libc6 (>= 2.7), alien-arena-data (>= 7.40)", "Filename": "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", "SHA1": " 46955e48cad27410a83740a21d766ce362364024", "SHA256": " eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", "Priority": "extra", "Maintainer": "Debian Games Team ", "Description": "Common files for Alien Arena client and server ALIEN ARENA is a standalone 3D first person online deathmatch shooter\n crafted from the original source code of Quake II and Quake III, released\n by id Software under the GPL license. With features including 32 bit\n graphics, new particle engine and effects, light blooms, reflective water,\n hi resolution textures and skins, hi poly models, stain maps, ALIEN ARENA\n pushes the envelope of graphical beauty rivaling today's top games.\n .\n This package installs the common files for Alien Arena.\n", "Homepage": "http://red.planetarena.org", "Tag": "role::app-data, role::shared-lib, special::auto-inst-parts", "Installed-Size": "456", "Version": "7.40-2", "Replaces": "alien-arena (<< 7.33-1)", "Size": "187518", "MD5sum": "1e8cba92c41420aa7baa8a5718d67122", "Package": "alien-arena-common", "Section": "contrib/games", "Architecture": "i386"} @@ -451,20 +337,20 @@ Architecture: all Standards-Version: 3.9.3 Format: 3.0 (quilt) Files: + ab56b4d92b40713acc5af89985d4b786 5 access-modifier-checker_1.0-4.debian.tar.gz 900150983cd24fb0d6963f7d28e17f72 3 access-modifier-checker_1.0-4.dsc e2fc714c4727ee9395f324cd2e7f331f 4 access-modifier-checker_1.0.orig.tar.gz - ab56b4d92b40713acc5af89985d4b786 5 access-modifier-checker_1.0-4.debian.tar.gz Dm-Upload-Allowed: yes Vcs-Browser: http://git.debian.org/?p=pkg-java/access-modifier-checker.git Vcs-Git: git://git.debian.org/git/pkg-java/access-modifier-checker.git Checksums-Sha1: + 03de6c570bfe24bfc328ccd7ca46b76eadaf4334 5 access-modifier-checker_1.0-4.debian.tar.gz a9993e364706816aba3e25717850c26c9cd0d89d 3 access-modifier-checker_1.0-4.dsc 81fe8bfe87576c3ecb22426f8e57847382917acf 4 access-modifier-checker_1.0.orig.tar.gz - 03de6c570bfe24bfc328ccd7ca46b76eadaf4334 5 access-modifier-checker_1.0-4.debian.tar.gz Checksums-Sha256: + 36bbe50ed96841d10443bcb670d6554f0a34b761be67ec9c4a8ad2c0c44ca42c 5 access-modifier-checker_1.0-4.debian.tar.gz ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad 3 access-modifier-checker_1.0-4.dsc 88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589 4 access-modifier-checker_1.0.orig.tar.gz - 36bbe50ed96841d10443bcb670d6554f0a34b761be67ec9c4a8ad2c0c44ca42c 5 access-modifier-checker_1.0-4.debian.tar.gz Homepage: https://github.com/kohsuke/access-modifier Package-List: libaccess-modifier-checker-java deb java optional diff --git a/debian/publish.go b/debian/publish.go index c8a4b3c5..63122a49 100644 --- a/debian/publish.go +++ b/debian/publish.go @@ -162,6 +162,10 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage return err } + pkg.files = nil + pkg.deps = nil + pkg.extra = nil + } return nil diff --git a/debian/publish_test.go b/debian/publish_test.go index 9e097f9a..cb53198b 100644 --- a/debian/publish_test.go +++ b/debian/publish_test.go @@ -77,7 +77,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) { s.packageCollection.Update(s.p2) s.packageCollection.Update(s.p3) - poolPath, _ := s.packagePool.Path(s.p1.Files[0].Filename, s.p1.Files[0].Checksums.MD5) + poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5) err := os.MkdirAll(filepath.Dir(poolPath), 0755) f, err := os.Create(poolPath) c.Assert(err, IsNil) diff --git a/debian/reflist.go b/debian/reflist.go new file mode 100644 index 00000000..3eeb7a46 --- /dev/null +++ b/debian/reflist.go @@ -0,0 +1,286 @@ +package debian + +import ( + "bytes" + "github.com/ugorji/go/codec" + "sort" +) + +// PackageRefList is a list of keys of packages, this is basis for snapshot +// and similar stuff +// +// Refs are sorted in lexicographical order +type PackageRefList struct { + // List of package keys + Refs [][]byte +} + +// Verify interface +var ( + _ sort.Interface = &PackageRefList{} +) + +// NewPackageRefList creates empty PackageRefList +func NewPackageRefList() *PackageRefList { + return &PackageRefList{} +} + +// NewPackageRefListFromPackageList creates PackageRefList from PackageList +func NewPackageRefListFromPackageList(list *PackageList) *PackageRefList { + reflist := &PackageRefList{} + reflist.Refs = make([][]byte, list.Len()) + + i := 0 + for _, p := range list.packages { + reflist.Refs[i] = p.Key("") + i++ + } + + sort.Sort(reflist) + + return reflist +} + +// Len returns number of refs +func (l *PackageRefList) Len() int { + return len(l.Refs) +} + +// Swap swaps two refs +func (l *PackageRefList) Swap(i, j int) { + l.Refs[i], l.Refs[j] = l.Refs[j], l.Refs[i] +} + +// Compare compares two refs in lexographical order +func (l *PackageRefList) Less(i, j int) bool { + return bytes.Compare(l.Refs[i], l.Refs[j]) < 0 +} + +// Encode does msgpack encoding of PackageRefList +func (l *PackageRefList) Encode() []byte { + var buf bytes.Buffer + + encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{}) + encoder.Encode(l) + + return buf.Bytes() +} + +// Decode decodes msgpack representation into PackageRefLit +func (l *PackageRefList) Decode(input []byte) error { + decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{}) + return decoder.Decode(l) +} + +// ForEach calls handler for each package ref in list +func (l *PackageRefList) ForEach(handler func([]byte) error) error { + var err error + for _, p := range l.Refs { + err = handler(p) + if err != nil { + return err + } + } + return err +} + +// Substract returns all packages in l that are not in r +func (l *PackageRefList) Substract(r *PackageRefList) *PackageRefList { + result := &PackageRefList{Refs: make([][]byte, 0, 128)} + + // pointer to left and right reflists + il, ir := 0, 0 + // length of reflists + ll, lr := l.Len(), r.Len() + + for il < ll || ir < lr { + if il == ll { + // left list exhausted, we got the result + break + } + if ir == lr { + // right list exhausted, append what is left to result + result.Refs = append(result.Refs, l.Refs[il:]...) + break + } + + rel := bytes.Compare(l.Refs[il], r.Refs[ir]) + if rel == 0 { + // r contains entry from l, so we skip it + il++ + ir++ + } else if rel < 0 { + // item il is not in r, append + result.Refs = append(result.Refs, l.Refs[il]) + il++ + } else { + // skip over to next item in r + ir++ + } + } + + return result +} + +// PackageDiff is a difference between two packages in a list. +// +// If left & right are present, difference is in package version +// If left is nil, package is present only in right +// If right is nil, package is present only in left +type PackageDiff struct { + Left, Right *Package +} + +// PackageDiffs is a list of PackageDiff records +type PackageDiffs []PackageDiff + +// Diff calculates difference between two reflists +func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageCollection) (result PackageDiffs, err error) { + result = make(PackageDiffs, 0, 128) + + // pointer to left and right reflists + il, ir := 0, 0 + // length of reflists + ll, lr := l.Len(), r.Len() + // cached loaded packages on the left & right + pl, pr := (*Package)(nil), (*Package)(nil) + + // until we reached end of both lists + for il < ll || ir < lr { + // if we've exhausted left list, pull the rest from the right + if il == ll { + pr, err = packageCollection.ByKey(r.Refs[ir]) + if err != nil { + return nil, err + } + result = append(result, PackageDiff{Left: nil, Right: pr}) + ir++ + continue + } + // if we've exhausted right list, pull the rest from the left + if ir == lr { + pl, err = packageCollection.ByKey(l.Refs[il]) + if err != nil { + return nil, err + } + result = append(result, PackageDiff{Left: pl, Right: nil}) + il++ + continue + } + + // refs on both sides are present, load them + rl, rr := l.Refs[il], r.Refs[ir] + // compare refs + rel := bytes.Compare(rl, rr) + + if rel == 0 { + // refs are identical, so are packages, advance pointer + il++ + ir++ + pl, pr = nil, nil + } else { + // load pl & pr if they haven't been loaded before + if pl == nil { + pl, err = packageCollection.ByKey(rl) + if err != nil { + return nil, err + } + } + + if pr == nil { + pr, err = packageCollection.ByKey(rr) + if err != nil { + return nil, err + } + } + + // is pl & pr the same package, but different version? + if pl.Name == pr.Name && pl.Architecture == pr.Architecture { + result = append(result, PackageDiff{Left: pl, Right: pr}) + il++ + ir++ + pl, pr = nil, nil + } else { + // otherwise pl or pr is missing on one of the sides + if rel < 0 { + result = append(result, PackageDiff{Left: pl, Right: nil}) + il++ + pl = nil + } else { + result = append(result, PackageDiff{Left: nil, Right: pr}) + ir++ + pr = nil + } + } + + } + } + + return +} + +// Merge merges reflist r into current reflist. If overrideMatching, merge replaces matching packages (by architecture/name) +// with reference from r, otherwise all packages are saved. +func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result *PackageRefList) { + // pointer to left and right reflists + il, ir := 0, 0 + // length of reflists + ll, lr := l.Len(), r.Len() + + result = &PackageRefList{} + result.Refs = make([][]byte, 0, ll+lr) + + // until we reached end of both lists + for il < ll || ir < lr { + // if we've exhausted left list, pull the rest from the right + if il == ll { + result.Refs = append(result.Refs, r.Refs[ir:]...) + break + } + // if we've exhausted right list, pull the rest from the left + if ir == lr { + result.Refs = append(result.Refs, l.Refs[il:]...) + break + } + + // refs on both sides are present, load them + rl, rr := l.Refs[il], r.Refs[ir] + // compare refs + rel := bytes.Compare(rl, rr) + + if rel == 0 { + // refs are identical, so are packages, advance pointer + result.Refs = append(result.Refs, l.Refs[il]) + il++ + ir++ + } else { + if overrideMatching { + partsL := bytes.Split(rl, []byte(" ")) + archL, nameL := partsL[0][1:], partsL[1] + + partsR := bytes.Split(rr, []byte(" ")) + archR, nameR := partsR[0][1:], partsR[1] + + if bytes.Compare(archL, archR) == 0 && bytes.Compare(nameL, nameR) == 0 { + // override with package from the right + result.Refs = append(result.Refs, r.Refs[ir]) + il++ + ir++ + continue + } + } + + // otherwise append smallest of two + if rel < 0 { + result.Refs = append(result.Refs, l.Refs[il]) + il++ + } else { + result.Refs = append(result.Refs, r.Refs[ir]) + ir++ + } + + } + } + + return +} diff --git a/debian/reflist_test.go b/debian/reflist_test.go new file mode 100644 index 00000000..52876b43 --- /dev/null +++ b/debian/reflist_test.go @@ -0,0 +1,275 @@ +package debian + +import ( + "errors" + "github.com/smira/aptly/database" + . "launchpad.net/gocheck" +) + +type PackageRefListSuite struct { + // Simple list with "real" packages from stanzas + list *PackageList + p1, p2, p3, p4, p5, p6 *Package +} + +var _ = Suite(&PackageRefListSuite{}) + +func (s *PackageRefListSuite) SetUpTest(c *C) { + s.list = NewPackageList() + + s.p1 = NewPackageFromControlFile(packageStanza.Copy()) + s.p2 = NewPackageFromControlFile(packageStanza.Copy()) + stanza := packageStanza.Copy() + stanza["Package"] = "mars-invaders" + s.p3 = NewPackageFromControlFile(stanza) + stanza = packageStanza.Copy() + stanza["Size"] = "42" + s.p4 = NewPackageFromControlFile(stanza) + stanza = packageStanza.Copy() + stanza["Package"] = "lonely-strangers" + s.p5 = NewPackageFromControlFile(stanza) + stanza = packageStanza.Copy() + stanza["Version"] = "99.1" + s.p6 = NewPackageFromControlFile(stanza) +} + +func (s *PackageRefListSuite) TestNewPackageListFromRefList(c *C) { + db, _ := database.OpenDB(c.MkDir()) + coll := NewPackageCollection(db) + coll.Update(s.p1) + coll.Update(s.p3) + + s.list.Add(s.p1) + s.list.Add(s.p3) + s.list.Add(s.p5) + s.list.Add(s.p6) + + reflist := NewPackageRefListFromPackageList(s.list) + + _, err := NewPackageListFromRefList(reflist, coll) + c.Assert(err, ErrorMatches, "unable to load package with key.*") + + coll.Update(s.p5) + coll.Update(s.p6) + + list, err := NewPackageListFromRefList(reflist, coll) + c.Assert(err, IsNil) + c.Check(list.Len(), Equals, 4) + c.Check(list.Add(s.p4), ErrorMatches, "conflict in package.*") + + list, err = NewPackageListFromRefList(nil, coll) + c.Assert(err, IsNil) + c.Check(list.Len(), Equals, 0) +} + +func (s *PackageRefListSuite) TestNewPackageRefList(c *C) { + s.list.Add(s.p1) + s.list.Add(s.p3) + s.list.Add(s.p5) + s.list.Add(s.p6) + + reflist := NewPackageRefListFromPackageList(s.list) + c.Assert(reflist.Len(), Equals, 4) + c.Check(reflist.Refs[0], DeepEquals, []byte(s.p1.Key(""))) + c.Check(reflist.Refs[1], DeepEquals, []byte(s.p6.Key(""))) + c.Check(reflist.Refs[2], DeepEquals, []byte(s.p5.Key(""))) + c.Check(reflist.Refs[3], DeepEquals, []byte(s.p3.Key(""))) + + reflist = NewPackageRefList() + c.Check(reflist.Len(), Equals, 0) +} + +func (s *PackageRefListSuite) TestPackageRefListEncodeDecode(c *C) { + s.list.Add(s.p1) + s.list.Add(s.p3) + s.list.Add(s.p5) + s.list.Add(s.p6) + + reflist := NewPackageRefListFromPackageList(s.list) + + reflist2 := &PackageRefList{} + err := reflist2.Decode(reflist.Encode()) + c.Assert(err, IsNil) + c.Check(reflist2.Len(), Equals, reflist.Len()) + c.Check(reflist2.Refs, DeepEquals, reflist.Refs) +} + +func (s *PackageRefListSuite) TestPackageRefListForeach(c *C) { + s.list.Add(s.p1) + s.list.Add(s.p3) + s.list.Add(s.p5) + s.list.Add(s.p6) + + reflist := NewPackageRefListFromPackageList(s.list) + + Len := 0 + err := reflist.ForEach(func([]byte) error { + Len++ + return nil + }) + + c.Check(Len, Equals, 4) + c.Check(err, IsNil) + + e := errors.New("b") + + err = reflist.ForEach(func([]byte) error { + return e + }) + + c.Check(err, Equals, e) +} + +func (s *PackageRefListSuite) TestSubstract(c *C) { + r1 := []byte("r1") + r2 := []byte("r2") + r3 := []byte("r3") + r4 := []byte("r4") + r5 := []byte("r5") + + empty := &PackageRefList{Refs: [][]byte{}} + l1 := &PackageRefList{Refs: [][]byte{r1, r2, r3, r4}} + l2 := &PackageRefList{Refs: [][]byte{r1, r3}} + l3 := &PackageRefList{Refs: [][]byte{r2, r4}} + l4 := &PackageRefList{Refs: [][]byte{r4, r5}} + l5 := &PackageRefList{Refs: [][]byte{r1, r2, r3}} + + c.Check(l1.Substract(empty), DeepEquals, l1) + c.Check(l1.Substract(l2), DeepEquals, l3) + c.Check(l1.Substract(l3), DeepEquals, l2) + c.Check(l1.Substract(l4), DeepEquals, l5) + c.Check(empty.Substract(l1), DeepEquals, empty) + c.Check(l2.Substract(l3), DeepEquals, l2) +} + +func (s *PackageRefListSuite) TestDiff(c *C) { + db, _ := database.OpenDB(c.MkDir()) + coll := NewPackageCollection(db) + + packages := []*Package{ + &Package{Name: "lib", Version: "1.0", Architecture: "i386"}, //0 + &Package{Name: "dpkg", Version: "1.7", Architecture: "i386"}, //1 + &Package{Name: "data", Version: "1.1~bp1", Architecture: "all"}, //2 + &Package{Name: "app", Version: "1.1~bp1", Architecture: "i386"}, //3 + &Package{Name: "app", Version: "1.1~bp2", Architecture: "i386"}, //4 + &Package{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5 + &Package{Name: "xyz", Version: "3.0", Architecture: "sparc"}, //6 + } + + for _, p := range packages { + coll.Update(p) + } + + listA := NewPackageList() + listA.Add(packages[0]) + listA.Add(packages[1]) + listA.Add(packages[2]) + listA.Add(packages[3]) + listA.Add(packages[6]) + + listB := NewPackageList() + listB.Add(packages[0]) + listB.Add(packages[2]) + listB.Add(packages[4]) + listB.Add(packages[5]) + + reflistA := NewPackageRefListFromPackageList(listA) + reflistB := NewPackageRefListFromPackageList(listB) + + diffAA, err := reflistA.Diff(reflistA, coll) + c.Check(err, IsNil) + c.Check(diffAA, HasLen, 0) + + diffAB, err := reflistA.Diff(reflistB, coll) + c.Check(err, IsNil) + c.Check(diffAB, HasLen, 4) + + c.Check(diffAB[0].Left, IsNil) + c.Check(diffAB[0].Right.String(), Equals, "app_1.1~bp2_amd64") + + c.Check(diffAB[1].Left.String(), Equals, "app_1.1~bp1_i386") + c.Check(diffAB[1].Right.String(), Equals, "app_1.1~bp2_i386") + + c.Check(diffAB[2].Left.String(), Equals, "dpkg_1.7_i386") + c.Check(diffAB[2].Right, IsNil) + + c.Check(diffAB[3].Left.String(), Equals, "xyz_3.0_sparc") + c.Check(diffAB[3].Right, IsNil) + + diffBA, err := reflistB.Diff(reflistA, coll) + c.Check(err, IsNil) + c.Check(diffBA, HasLen, 4) + + c.Check(diffBA[0].Right, IsNil) + c.Check(diffBA[0].Left.String(), Equals, "app_1.1~bp2_amd64") + + c.Check(diffBA[1].Right.String(), Equals, "app_1.1~bp1_i386") + c.Check(diffBA[1].Left.String(), Equals, "app_1.1~bp2_i386") + + c.Check(diffBA[2].Right.String(), Equals, "dpkg_1.7_i386") + c.Check(diffBA[2].Left, IsNil) + + c.Check(diffBA[3].Right.String(), Equals, "xyz_3.0_sparc") + c.Check(diffBA[3].Left, IsNil) + +} + +func (s *PackageRefListSuite) TestMerge(c *C) { + db, _ := database.OpenDB(c.MkDir()) + coll := NewPackageCollection(db) + + packages := []*Package{ + &Package{Name: "lib", Version: "1.0", Architecture: "i386"}, //0 + &Package{Name: "dpkg", Version: "1.7", Architecture: "i386"}, //1 + &Package{Name: "data", Version: "1.1~bp1", Architecture: "all"}, //2 + &Package{Name: "app", Version: "1.1~bp1", Architecture: "i386"}, //3 + &Package{Name: "app", Version: "1.1~bp2", Architecture: "i386"}, //4 + &Package{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5 + &Package{Name: "dpkg", Version: "1.0", Architecture: "i386"}, //6 + &Package{Name: "xyz", Version: "1.0", Architecture: "sparc"}, //7 + } + + for _, p := range packages { + coll.Update(p) + } + + listA := NewPackageList() + listA.Add(packages[0]) + listA.Add(packages[1]) + listA.Add(packages[2]) + listA.Add(packages[3]) + listA.Add(packages[7]) + + listB := NewPackageList() + listB.Add(packages[0]) + listB.Add(packages[2]) + listB.Add(packages[4]) + listB.Add(packages[5]) + listB.Add(packages[6]) + + reflistA := NewPackageRefListFromPackageList(listA) + reflistB := NewPackageRefListFromPackageList(listB) + + toStrSlice := func(reflist *PackageRefList) (result []string) { + result = make([]string, reflist.Len()) + for i, r := range reflist.Refs { + result[i] = string(r) + } + return + } + + mergeAB := reflistA.Merge(reflistB, true) + mergeBA := reflistB.Merge(reflistA, true) + + c.Check(toStrSlice(mergeAB), DeepEquals, + []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 lib 1.0", "Psparc xyz 1.0"}) + c.Check(toStrSlice(mergeBA), DeepEquals, + []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"}) + + mergeABall := reflistA.Merge(reflistB, false) + mergeBAall := reflistB.Merge(reflistA, false) + + c.Check(mergeABall, DeepEquals, mergeBAall) + c.Check(toStrSlice(mergeBAall), DeepEquals, + []string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"}) +} diff --git a/debian/remote.go b/debian/remote.go index ab162849..38de801e 100644 --- a/debian/remote.go +++ b/debian/remote.go @@ -363,54 +363,31 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, pa return err } } - list.Add(p) - } - } - - progress.Printf("Saving packages to database...\n") - - progress.InitBar(int64(list.Len()), false) - - packageCollection.db.StartBatch() - count := 0 - - // Save package meta information to DB - err := list.ForEach(func(p *Package) error { - progress.AddBar(1) - count++ - if count > 1000 { - count = 0 - err := packageCollection.db.FinishBatch() + err = list.Add(p) + if err != nil { + return err + } + + err = packageCollection.Update(p) if err != nil { return err } - packageCollection.db.StartBatch() } - return packageCollection.Update(p) - }) - if err != nil { - return fmt.Errorf("unable to save packages to db: %s", err) } - err = packageCollection.db.FinishBatch() - if err != nil { - return fmt.Errorf("unable to save packages to db: %s", err) - } - - progress.ShutdownBar() - progress.Printf("Building download queue...\n") // Build download queue queued := make(map[string]PackageDownloadTask, list.Len()) - count = 0 + count := 0 downloadSize := int64(0) - err = list.ForEach(func(p *Package) error { + err := list.ForEach(func(p *Package) error { list, err := p.DownloadList(packagePool) if err != nil { return err } + p.files = nil for _, task := range list { key := task.RepoURI + "-" + task.DestinationPath diff --git a/debian/remote_test.go b/debian/remote_test.go index e19b3ccf..a1b1b6a2 100644 --- a/debian/remote_test.go +++ b/debian/remote_test.go @@ -535,7 +535,7 @@ MD5Sum: 4059d198768f9f8dc9372dc1c54bc3c3 14 main/debian-installer/binary-powerpc/Packages.bz2 9d10bb61e59bd799891ae4fbcf447ec9 29 main/debian-installer/binary-powerpc/Packages.gz 3481d65651306df1596dca9078c2506a 135 main/source/Release - dd09503813f711817457551858f6ec2f 2003 main/source/Sources + 0459b7e4512db5479cb982bac6e2f9a1 2003 main/source/Sources 3d83a489f1bd3c04226aa6520b8a6d07 656 main/source/Sources.bz2 b062b5b77094aeeb05ca8dbb1ecf68a9 592 main/source/Sources.gz SHA1: @@ -575,7 +575,7 @@ SHA1: 64a543afbb5f4bf728636bdcbbe7a2ed0804adc2 14 main/debian-installer/binary-powerpc/Packages.bz2 3df6ca52b6e8ecfb4a8fac6b8e02c777e3c7960d 29 main/debian-installer/binary-powerpc/Packages.gz 49cfec0c9b1df3a25e983a3ddf29d15b0e376e02 135 main/source/Release - dfa81a35ac2eebcefdc51ae73486c480e277ef1b 2003 main/source/Sources + 6b92e0fc84307226172696fde59ca5f33f380b57 2003 main/source/Sources ecb8afea11030a5df46941cb8ec297ca24c85736 656 main/source/Sources.bz2 923e71383969c91146f12fa8cd121397f2467a2e 592 main/source/Sources.gz SHA256: @@ -615,7 +615,7 @@ SHA256: d3dda84eb03b9738d118eb2be78e246106900493c0ae07819ad60815134a8058 14 main/debian-installer/binary-powerpc/Packages.bz2 825d493158fe0f50ca1acd70367aefa391170563af2e4ee9cedbcbe6796c8384 29 main/debian-installer/binary-powerpc/Packages.gz d683102993b6f11067ce86d73111f067e36a199e9dc1f4295c8b19c274dc9ef8 135 main/source/Release - 4d911b084c12c71ce973743a3c27fc4efd18d7b8faa6ee936add9cca8e2c6ccf 2003 main/source/Sources + 45f868fd5d9efe611d67572ffcf96a00a5b9ec38ea5102753290c38c36b8c282 2003 main/source/Sources d178f1e310218d9f0f16c37d0780637f1cf3640a94a7fb0e24dc940c51b1e115 656 main/source/Sources.bz2 080228b550da407fb8ac73fb30b37323468fd2b2de98dd56a324ee7d701f6103 592 main/source/Sources.gz`