Store package ref list in separate entity and load it only on demand.

debian/list.go

@@ -3,6 +3,7 @@ package debian
 import (
 	"bytes"
 	"fmt"
+	"github.com/ugorji/go/codec"
 	"sort"
 )
 
@@ -86,3 +87,19 @@ func (l *PackageRefList) Swap(i, j int) {
 func (l *PackageRefList) Less(i, j int) bool {
 	return bytes.Compare(l.Refs[i], l.Refs[j]) < 0
 }
+
+// Encode does msgpack encoding of PackageRefList
+func (l *PackageRefList) Encode() []byte {
+	var buf bytes.Buffer
+
+	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
+	encoder.Encode(l)
+
+	return buf.Bytes()
+}
+
+// Decode decodes msgpack representation into PackageRefLit
+func (l *PackageRefList) Decode(input []byte) error {
+	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
+	return decoder.Decode(l)
+}

debian/list_test.go

@@ -6,8 +6,8 @@ import (
 )
 
 type PackageListSuite struct {
-	list           *PackageList
-	p1, p2, p3, p4 *Package
+	list                   *PackageList
+	p1, p2, p3, p4, p5, p6 *Package
 }
 
 var _ = Suite(&PackageListSuite{})
@@ -31,6 +31,12 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 	para = paraGen()
 	para["Size"] = "42"
 	s.p4 = NewPackageFromControlFile(para)
+	para = paraGen()
+	para["Package"] = "lonely-strangers"
+	s.p5 = NewPackageFromControlFile(para)
+	para = paraGen()
+	para["Version"] = "99.1"
+	s.p6 = NewPackageFromControlFile(para)
 }
 
 func (s *PackageListSuite) TestAddLen(c *C) {
@@ -59,9 +65,28 @@ func (s *PackageListSuite) TestForeach(c *C) {
 func (s *PackageListSuite) TestNewPackageRefList(c *C) {
 	s.list.Add(s.p1)
 	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	s.list.Add(s.p6)
 
 	reflist := NewPackageRefListFromPackageList(s.list)
-	c.Assert(reflist.Len(), Equals, 2)
-	c.Assert(reflist.Refs[0], DeepEquals, []byte(s.p1.Key()))
-	c.Assert(reflist.Refs[1], DeepEquals, []byte(s.p3.Key()))
+	c.Assert(reflist.Len(), Equals, 4)
+	c.Check(reflist.Refs[0], DeepEquals, []byte(s.p1.Key()))
+	c.Check(reflist.Refs[1], DeepEquals, []byte(s.p6.Key()))
+	c.Check(reflist.Refs[2], DeepEquals, []byte(s.p5.Key()))
+	c.Check(reflist.Refs[3], DeepEquals, []byte(s.p3.Key()))
+}
+
+func (s *PackageListSuite) TestPackageRefListEncodeDecode(c *C) {
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	s.list.Add(s.p6)
+
+	reflist := NewPackageRefListFromPackageList(s.list)
+
+	reflist2 := &PackageRefList{}
+	err := reflist2.Decode(reflist.Encode())
+	c.Assert(err, IsNil)
+	c.Check(reflist2.Len(), Equals, reflist.Len())
+	c.Check(reflist2.Refs, DeepEquals, reflist.Refs)
 }

debian/remote.go

@@ -37,7 +37,8 @@ type RemoteRepo struct {
 	// Last update date
 	LastDownloadDate time.Time
 	// "Snapshot" of current list of packages
-	PackageRefs    *PackageRefList
+	packageRefs *PackageRefList
+	// Parsed archived root
 	archiveRootURL *url.URL
 }
 
@@ -70,6 +71,14 @@ func (repo *RemoteRepo) String() string {
 	return fmt.Sprintf("[%s]: %s %s", repo.Name, repo.ArchiveRoot, repo.Distribution)
 }
 
+// NumPackages return number of packages retrived from remore repo
+func (repo *RemoteRepo) NumPackages() int {
+	if repo.packageRefs == nil {
+		return 0
+	}
+	return repo.packageRefs.Len()
+}
+
 // ReleaseURL returns URL to Release file in repo root
 // TODO: InRelease, Release.gz, Release.bz2 handling
 func (repo *RemoteRepo) ReleaseURL() *url.URL {
@@ -194,7 +203,7 @@ func (repo *RemoteRepo) Download(d utils.Downloader, db database.Storage, packag
 	}
 
 	repo.LastDownloadDate = time.Now()
-	repo.PackageRefs = NewPackageRefListFromPackageList(list)
+	repo.packageRefs = NewPackageRefListFromPackageList(list)
 
 	return nil
 }
@@ -224,6 +233,11 @@ func (repo *RemoteRepo) Key() []byte {
 	return []byte("R" + repo.UUID)
 }
 
+// RefKey is a unique id for package reference list
+func (repo *RemoteRepo) RefKey() []byte {
+	return []byte("E" + repo.UUID)
+}
+
 // RemoteRepoCollection does listing, updating/adding/deleting of RemoteRepos
 type RemoteRepoCollection struct {
 	db   database.Storage
@@ -270,7 +284,31 @@ func (collection *RemoteRepoCollection) Add(repo *RemoteRepo) error {
 
 // Update stores updated information about repo in DB
 func (collection *RemoteRepoCollection) Update(repo *RemoteRepo) error {
-	return collection.db.Put(repo.Key(), repo.Encode())
+	err := collection.db.Put(repo.Key(), repo.Encode())
+	if err != nil {
+		return err
+	}
+	if repo.packageRefs != nil {
+		err = collection.db.Put(repo.RefKey(), repo.packageRefs.Encode())
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// LoadComplete loads additional information for remote repo
+func (collection *RemoteRepoCollection) LoadComplete(repo *RemoteRepo) error {
+	encoded, err := collection.db.Get(repo.RefKey())
+	if err == database.ErrNotFound {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	repo.packageRefs = &PackageRefList{}
+	return repo.packageRefs.Decode(encoded)
 }
 
 // ByName looks up repository by name

debian/remote_test.go

@@ -3,6 +3,7 @@ package debian
 import (
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/utils"
+	debc "github.com/smira/godebiancontrol"
 	. "launchpad.net/gocheck"
 	"testing"
 )
@@ -12,7 +13,40 @@ func Test(t *testing.T) {
 	TestingT(t)
 }
 
+type PackageListMixinSuite struct {
+	p1, p2, p3 *Package
+	list       *PackageList
+	reflist    *PackageRefList
+}
+
+func (s *PackageListMixinSuite) SetUpPackages() {
+	s.list = NewPackageList()
+
+	paraGen := func() debc.Paragraph {
+		para := make(debc.Paragraph)
+		for k, v := range packagePara {
+			para[k] = v
+		}
+		return para
+	}
+
+	s.p1 = NewPackageFromControlFile(paraGen())
+	para := paraGen()
+	para["Package"] = "mars-invaders"
+	s.p2 = NewPackageFromControlFile(para)
+	para = paraGen()
+	para["Package"] = "lonely-strangers"
+	s.p3 = NewPackageFromControlFile(para)
+
+	s.list.Add(s.p1)
+	s.list.Add(s.p2)
+	s.list.Add(s.p3)
+
+	s.reflist = NewPackageRefListFromPackageList(s.list)
+}
+
 type RemoteRepoSuite struct {
+	PackageListMixinSuite
 	repo       *RemoteRepo
 	downloader utils.Downloader
 }
@@ -22,6 +56,7 @@ var _ = Suite(&RemoteRepoSuite{})
 func (s *RemoteRepoSuite) SetUpTest(c *C) {
 	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{})
 	s.downloader = utils.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+	s.SetUpPackages()
 }
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
@@ -29,6 +64,12 @@ func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
 	c.Assert(err, ErrorMatches, ".*hexadecimal escape in host.*")
 }
 
+func (s *RemoteRepoSuite) TestNumPackages(c *C) {
+	c.Check(s.repo.NumPackages(), Equals, 0)
+	s.repo.packageRefs = s.reflist
+	c.Check(s.repo.NumPackages(), Equals, 3)
+}
+
 func (s *RemoteRepoSuite) TestReleaseURL(c *C) {
 	c.Assert(s.repo.ReleaseURL().String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/Release")
 }
@@ -72,9 +113,16 @@ func (s *RemoteRepoSuite) TestEncodeDecode(c *C) {
 
 func (s *RemoteRepoSuite) TestKey(c *C) {
 	c.Assert(len(s.repo.Key()), Equals, 37)
+	c.Assert(s.repo.Key()[0], Equals, byte('R'))
+}
+
+func (s *RemoteRepoSuite) TestRefKey(c *C) {
+	c.Assert(len(s.repo.RefKey()), Equals, 37)
+	c.Assert(s.repo.RefKey()[0], Equals, byte('E'))
 }
 
 type RemoteRepoCollectionSuite struct {
+	PackageListMixinSuite
 	db         database.Storage
 	collection *RemoteRepoCollection
 }
@@ -84,6 +132,7 @@ var _ = Suite(&RemoteRepoCollectionSuite{})
 func (s *RemoteRepoCollectionSuite) SetUpTest(c *C) {
 	s.db, _ = database.OpenDB(c.MkDir())
 	s.collection = NewRemoteRepoCollection(s.db)
+	s.SetUpPackages()
 }
 
 func (s *RemoteRepoCollectionSuite) TearDownTest(c *C) {
@@ -109,6 +158,27 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 
 }
 
+func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{})
+	c.Assert(s.collection.Update(repo), IsNil)
+
+	collection := NewRemoteRepoCollection(s.db)
+	r, err := collection.ByName("yandex")
+	c.Assert(err, IsNil)
+	c.Assert(r.packageRefs, IsNil)
+
+	repo.packageRefs = s.reflist
+	c.Assert(s.collection.Update(repo), IsNil)
+
+	collection = NewRemoteRepoCollection(s.db)
+	r, err = collection.ByName("yandex")
+	c.Assert(err, IsNil)
+	c.Assert(r.packageRefs, IsNil)
+	c.Assert(r.NumPackages(), Equals, 0)
+	c.Assert(s.collection.LoadComplete(r), IsNil)
+	c.Assert(r.NumPackages(), Equals, 3)
+}
+
 func (s *RemoteRepoCollectionSuite) TestForEach(c *C) {
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{})
 	s.collection.Add(repo)