From 7ad1bb387b78cd86d23ad726c71026239cb2e249 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 25 Sep 2014 19:34:16 +0400 Subject: [PATCH] Support for .udeb downloads from remote mirrors. #108 --- deb/publish_test.go | 2 +- deb/remote.go | 25 +++++++++++++++++++++++-- deb/remote_test.go | 33 +++++++++++++++++++-------------- deb/snapshot_test.go | 8 ++++---- 4 files changed, 47 insertions(+), 21 deletions(-) diff --git a/deb/publish_test.go b/deb/publish_test.go index c361f241..56e1db95 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -93,7 +93,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) { "files:other": s.publishedStorage2}} s.packagePool = files.NewPackagePool(s.root) - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) repo.packageRefs = s.reflist s.factory.RemoteRepoCollection().Add(repo) diff --git a/deb/remote.go b/deb/remote.go index 1d439d44..fb4dceda 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -37,6 +37,8 @@ type RemoteRepo struct { Architectures []string // Should we download sources? DownloadSources bool + // Should we download .udebs? + DownloadUdebs bool // Meta-information about repository Meta Stanza // Last update date @@ -55,7 +57,7 @@ type RemoteRepo struct { // NewRemoteRepo creates new instance of Debian remote repository with specified params func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string, - architectures []string, downloadSources bool) (*RemoteRepo, error) { + architectures []string, downloadSources bool, downloadUdebs bool) (*RemoteRepo, error) { result := &RemoteRepo{ UUID: uuid.New(), Name: name, @@ -64,6 +66,7 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone Components: components, Architectures: architectures, DownloadSources: downloadSources, + DownloadUdebs: downloadUdebs, } err := result.prepare() @@ -80,6 +83,9 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone if len(result.Components) > 0 { return nil, fmt.Errorf("components aren't supported for flat repos") } + if result.DownloadUdebs { + return nil, fmt.Errorf("debian-installer udebs aren't supported for flat repos") + } result.Components = nil } @@ -102,7 +108,10 @@ func (repo *RemoteRepo) prepare() error { func (repo *RemoteRepo) String() string { srcFlag := "" if repo.DownloadSources { - srcFlag = " [src]" + srcFlag += " [src]" + } + if repo.DownloadUdebs { + srcFlag += " [udeb]" } distribution := repo.Distribution if distribution == "" { @@ -169,6 +178,13 @@ func (repo *RemoteRepo) SourcesURL(component string) *url.URL { return repo.archiveRootURL.ResolveReference(path) } +// UdebURL returns URL of Packages files for given component and +// architecture +func (repo *RemoteRepo) UdebURL(component string, architecture string) *url.URL { + path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/debian-installer/binary-%s/Packages", repo.Distribution, component, architecture)} + return repo.archiveRootURL.ResolveReference(path) +} + // PackageURL returns URL of package file relative to repository root // architecture func (repo *RemoteRepo) PackageURL(filename string) *url.URL { @@ -342,6 +358,9 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, co for _, component := range repo.Components { for _, architecture := range repo.Architectures { packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"}) + if repo.DownloadUdebs { + packagesURLs = append(packagesURLs, []string{repo.UdebURL(component, architecture).String(), "udeb"}) + } } if repo.DownloadSources { packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"}) @@ -378,6 +397,8 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, co if kind == "binary" { p = NewPackageFromControlFile(stanza) + } else if kind == "udeb" { + p = NewUdebPackageFromControlFile(stanza) } else if kind == "source" { p, err = NewSourcePackageFromControlFile(stanza) if err != nil { diff --git a/deb/remote_test.go b/deb/remote_test.go index 206101dc..af618d1c 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -79,8 +79,8 @@ type RemoteRepoSuite struct { var _ = Suite(&RemoteRepoSuite{}) func (s *RemoteRepoSuite) SetUpTest(c *C) { - s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false) - s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false) + s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false) + s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false) s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile) s.progress = console.NewProgress() s.db, _ = database.OpenDB(c.MkDir()) @@ -96,7 +96,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) { } func (s *RemoteRepoSuite) TestInvalidURL(c *C) { - _, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false) + _, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false) c.Assert(err, ErrorMatches, ".*hexadecimal escape in host.*") } @@ -106,11 +106,11 @@ func (s *RemoteRepoSuite) TestFlatCreation(c *C) { c.Check(s.flat.Architectures, IsNil) c.Check(s.flat.Components, IsNil) - flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false) + flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false) c.Check(flat2.IsFlat(), Equals, true) c.Check(flat2.Distribution, Equals, "./binary/") - _, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false) + _, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false) c.Check(err, ErrorMatches, "components aren't supported for flat repos") } @@ -119,8 +119,9 @@ func (s *RemoteRepoSuite) TestString(c *C) { c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./") s.repo.DownloadSources = true + s.repo.DownloadUdebs = true s.flat.DownloadSources = true - c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src]") + c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb]") c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./ [src]") } @@ -151,6 +152,10 @@ func (s *RemoteRepoSuite) TestBinaryURL(c *C) { c.Assert(s.repo.BinaryURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/binary-amd64/Packages") } +func (s *RemoteRepoSuite) TestUdebURL(c *C) { + c.Assert(s.repo.UdebURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/debian-installer/binary-amd64/Packages") +} + func (s *RemoteRepoSuite) TestSourcesURL(c *C) { c.Assert(s.repo.SourcesURL("main").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources") } @@ -209,13 +214,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) { } func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) { - s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false) + s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false) err := s.repo.Fetch(s.downloader, nil) c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*") } func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) { - s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false) + s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false) err := s.repo.Fetch(s.downloader, nil) c.Assert(err, ErrorMatches, "component xyz not available in repo.*") } @@ -399,7 +404,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) { r, err := s.collection.ByName("yandex") c.Assert(err, ErrorMatches, "*.not found") - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) c.Assert(s.collection.Add(repo), IsNil) c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists") @@ -417,7 +422,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) { r, err := s.collection.ByUUID("some-uuid") c.Assert(err, ErrorMatches, "*.not found") - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) c.Assert(s.collection.Add(repo), IsNil) r, err = s.collection.ByUUID(repo.UUID) @@ -426,7 +431,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) { } func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) { - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) c.Assert(s.collection.Update(repo), IsNil) collection := NewRemoteRepoCollection(s.db) @@ -447,7 +452,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) { } func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) { - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) s.collection.Add(repo) count := 0 @@ -469,10 +474,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) { } func (s *RemoteRepoCollectionSuite) TestDrop(c *C) { - repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false) + repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) s.collection.Add(repo1) - repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false) + repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false) s.collection.Add(repo2) r1, _ := s.collection.ByUUID(repo1.UUID) diff --git a/deb/snapshot_test.go b/deb/snapshot_test.go index 9d52d713..83db0853 100644 --- a/deb/snapshot_test.go +++ b/deb/snapshot_test.go @@ -15,7 +15,7 @@ var _ = Suite(&SnapshotSuite{}) func (s *SnapshotSuite) SetUpTest(c *C) { s.SetUpPackages() - s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false) + s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) s.repo.packageRefs = s.reflist } @@ -108,11 +108,11 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) { s.collection = NewSnapshotCollection(s.db) s.SetUpPackages() - s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false) + s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) s.repo1.packageRefs = s.reflist s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1) - s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false) + s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false) s.repo2.packageRefs = s.reflist s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2) @@ -192,7 +192,7 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) { c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1}) c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2}) - repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false) + repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false) c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot{}) }