From 5dd11a2ec26d4c019db5eef714627fc113fbb85b Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 20 Apr 2017 00:16:23 +0300 Subject: [PATCH] Pull original packages when skipping existing packages --- cmd/mirror_update.go | 4 +++- deb/remote.go | 50 +++++++++++++++++++++++++------------------- deb/remote_test.go | 24 ++++++++++----------- 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/cmd/mirror_update.go b/cmd/mirror_update.go index bde5df84..b5cb9311 100644 --- a/cmd/mirror_update.go +++ b/cmd/mirror_update.go @@ -86,7 +86,9 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool) context.Progress().Printf("Building download queue...\n") - queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), context.CollectionFactory().ChecksumCollection(), skipExistingPackages) + queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), context.CollectionFactory().PackageCollection(), + context.CollectionFactory().ChecksumCollection(), skipExistingPackages) + if err != nil { return fmt.Errorf("unable to update: %s", err) } diff --git a/deb/remote.go b/deb/remote.go index f75cb3e1..a0f1ae6d 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -501,35 +501,43 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu } // BuildDownloadQueue builds queue, discards current PackageList -func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) { +func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) { queue = make([]PackageDownloadTask, 0, repo.packageList.Len()) seen := make(map[string]int, repo.packageList.Len()) err = repo.packageList.ForEach(func(p *Package) error { - download := true if repo.packageRefs != nil && skipExistingPackages { - download = !repo.packageRefs.Has(p) - } - - if download { - list, err2 := p.DownloadList(packagePool, checksumStorage) - if err2 != nil { - return err2 - } - - for _, task := range list { - key := task.File.DownloadURL() - idx, found := seen[key] - if !found { - queue = append(queue, task) - downloadSize += task.File.Checksums.Size - seen[key] = len(queue) - 1 - } else { - // hook up the task to duplicate entry already on the list - queue[idx].Additional = append(queue[idx].Additional, task) + if repo.packageRefs.Has(p) { + // skip this package, but load checksums/files from package in DB + var prevP *Package + prevP, err = packageCollection.ByKey(p.Key("")) + if err != nil { + return err } + + p.UpdateFiles(prevP.Files()) + return nil } } + + list, err2 := p.DownloadList(packagePool, checksumStorage) + if err2 != nil { + return err2 + } + + for _, task := range list { + key := task.File.DownloadURL() + idx, found := seen[key] + if !found { + queue = append(queue, task) + downloadSize += task.File.Checksums.Size + seen[key] = len(queue) - 1 + } else { + // hook up the task to duplicate entry already on the list + queue[idx].Additional = append(queue[idx].Additional, task) + } + } + return nil }) if err != nil { diff --git a/deb/remote_test.go b/deb/remote_test.go index 5152d4ae..083367e3 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -268,7 +268,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.cs, false) + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -295,7 +295,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.cs, true) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -316,7 +316,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.cs, false) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -344,7 +344,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.cs, false) + queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) @@ -388,7 +388,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.cs, true) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -413,7 +413,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) - queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.cs, false) + queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) @@ -437,7 +437,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.cs, false) + queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -465,7 +465,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.cs, true) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -487,7 +487,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.cs, false) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(3)) c.Check(queue, HasLen, 1) @@ -518,7 +518,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.cs, false) + queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4) @@ -564,7 +564,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.cs, true) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true) c.Assert(err, IsNil) c.Check(size, Equals, int64(0)) c.Check(queue, HasLen, 0) @@ -590,7 +590,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) - queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.cs, false) + queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) c.Assert(err, IsNil) c.Check(size, Equals, int64(15)) c.Check(queue, HasLen, 4)