From 3075addd46c75dc68b29428f0790a6f0ae191198 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Mon, 10 Feb 2014 14:47:52 +0400 Subject: [PATCH] Support for flat repositories. --- debian/remote.go | 118 ++++++++++++++++++++++++++++-------------- debian/remote_test.go | 48 +++++++++++++++++ 2 files changed, 127 insertions(+), 39 deletions(-) diff --git a/debian/remote.go b/debian/remote.go index 9545b28e..ede3862c 100644 --- a/debian/remote.go +++ b/debian/remote.go @@ -60,6 +60,17 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone if err != nil { return nil, err } + + if result.Distribution == "." || result.Distribution == "./" { + // flat repo + result.Distribution = "" + result.Architectures = nil + if len(result.Components) > 0 { + return nil, fmt.Errorf("components aren't supported for flat repos") + } + result.Components = nil + } + return result, nil } @@ -74,6 +85,11 @@ func (repo *RemoteRepo) String() string { return fmt.Sprintf("[%s]: %s %s", repo.Name, repo.ArchiveRoot, repo.Distribution) } +// IsFlat determines if repository is flat +func (repo *RemoteRepo) IsFlat() bool { + return repo.Distribution == "" +} + // NumPackages return number of packages retrived from remore repo func (repo *RemoteRepo) NumPackages() int { if repo.packageRefs == nil { @@ -89,7 +105,20 @@ func (repo *RemoteRepo) RefList() *PackageRefList { // ReleaseURL returns URL to Release* files in repo root func (repo *RemoteRepo) ReleaseURL(name string) *url.URL { - path := &url.URL{Path: fmt.Sprintf("dists/%s/%s", repo.Distribution, name)} + var path *url.URL + + if !repo.IsFlat() { + path = &url.URL{Path: fmt.Sprintf("dists/%s/%s", repo.Distribution, name)} + } else { + path = &url.URL{Path: name} + } + + return repo.archiveRootURL.ResolveReference(path) +} + +// FlatBinaryURL returns URL to Package files for flat repo +func (repo *RemoteRepo) FlatBinaryURL() *url.URL { + path := &url.URL{Path: "Packages"} return repo.archiveRootURL.ResolveReference(path) } @@ -167,25 +196,27 @@ ok: return err } - architectures := strings.Split(stanza["Architectures"], " ") - if len(repo.Architectures) == 0 { - repo.Architectures = architectures - } else { - err = utils.StringsIsSubset(repo.Architectures, architectures, - fmt.Sprintf("architecture %%s not available in repo %s", repo)) - if err != nil { - return err + if !repo.IsFlat() { + architectures := strings.Split(stanza["Architectures"], " ") + if len(repo.Architectures) == 0 { + repo.Architectures = architectures + } else { + err = utils.StringsIsSubset(repo.Architectures, architectures, + fmt.Sprintf("architecture %%s not available in repo %s", repo)) + if err != nil { + return err + } } - } - components := strings.Split(stanza["Components"], " ") - if len(repo.Components) == 0 { - repo.Components = components - } else { - err = utils.StringsIsSubset(repo.Components, components, - fmt.Sprintf("component %%s not available in repo %s", repo)) - if err != nil { - return err + components := strings.Split(stanza["Components"], " ") + if len(repo.Components) == 0 { + repo.Components = components + } else { + err = utils.StringsIsSubset(repo.Components, components, + fmt.Sprintf("component %%s not available in repo %s", repo)) + if err != nil { + return err + } } } @@ -247,31 +278,40 @@ func (repo *RemoteRepo) Download(d utils.Downloader, packageCollection *PackageC d.GetProgress().Printf("Downloading & parsing package files...\n") - // Download and parse all Release files - for _, component := range repo.Components { - for _, architecture := range repo.Architectures { - packagesReader, packagesFile, err := utils.DownloadTryCompression(d, - repo.BinaryURL(component, architecture).String(), repo.ReleaseFiles, ignoreMismatch) + // Download and parse all Packages files + packagesURLs := []string{} + + if repo.IsFlat() { + packagesURLs = append(packagesURLs, repo.FlatBinaryURL().String()) + } else { + for _, component := range repo.Components { + for _, architecture := range repo.Architectures { + packagesURLs = append(packagesURLs, repo.BinaryURL(component, architecture).String()) + } + } + } + + for _, url := range packagesURLs { + packagesReader, packagesFile, err := utils.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch) + if err != nil { + return err + } + defer packagesFile.Close() + + sreader := NewControlFileReader(packagesReader) + + for { + stanza, err := sreader.ReadStanza() if err != nil { return err } - defer packagesFile.Close() - - sreader := NewControlFileReader(packagesReader) - - for { - stanza, err := sreader.ReadStanza() - if err != nil { - return err - } - if stanza == nil { - break - } - - p := NewPackageFromControlFile(stanza) - - list.Add(p) + if stanza == nil { + break } + + p := NewPackageFromControlFile(stanza) + + list.Add(p) } } diff --git a/debian/remote_test.go b/debian/remote_test.go index ad509072..4870923f 100644 --- a/debian/remote_test.go +++ b/debian/remote_test.go @@ -66,6 +66,7 @@ func (s *PackageListMixinSuite) SetUpPackages() { type RemoteRepoSuite struct { PackageListMixinSuite repo *RemoteRepo + flat *RemoteRepo downloader *utils.FakeDownloader db database.Storage packageCollection *PackageCollection @@ -76,6 +77,7 @@ var _ = Suite(&RemoteRepoSuite{}) func (s *RemoteRepoSuite) SetUpTest(c *C) { s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}) + s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}) s.downloader = utils.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile) s.db, _ = database.OpenDB(c.MkDir()) s.packageCollection = NewPackageCollection(s.db) @@ -92,12 +94,26 @@ func (s *RemoteRepoSuite) TestInvalidURL(c *C) { c.Assert(err, ErrorMatches, ".*hexadecimal escape in host.*") } +func (s *RemoteRepoSuite) TestFlatCreation(c *C) { + c.Check(s.flat.Distribution, Equals, "") + c.Check(s.flat.Architectures, IsNil) + c.Check(s.flat.Components, IsNil) + + _, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}) + c.Check(err, ErrorMatches, "components aren't supported for flat repos") +} + func (s *RemoteRepoSuite) TestNumPackages(c *C) { c.Check(s.repo.NumPackages(), Equals, 0) s.repo.packageRefs = s.reflist c.Check(s.repo.NumPackages(), Equals, 3) } +func (s *RemoteRepoSuite) TestIsFlat(c *C) { + c.Check(s.repo.IsFlat(), Equals, false) + c.Check(s.flat.IsFlat(), Equals, true) +} + func (s *RemoteRepoSuite) TestRefList(c *C) { s.repo.packageRefs = s.reflist c.Check(s.repo.RefList(), Equals, s.reflist) @@ -106,12 +122,18 @@ func (s *RemoteRepoSuite) TestRefList(c *C) { func (s *RemoteRepoSuite) TestReleaseURL(c *C) { c.Assert(s.repo.ReleaseURL("Release").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/Release") c.Assert(s.repo.ReleaseURL("InRelease").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/InRelease") + + c.Assert(s.flat.ReleaseURL("Release").String(), Equals, "http://repos.express42.com/virool/precise/Release") } func (s *RemoteRepoSuite) TestBinaryURL(c *C) { c.Assert(s.repo.BinaryURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/binary-amd64/Packages") } +func (s *RemoteRepoSuite) TestFlatBinaryURL(c *C) { + c.Assert(s.flat.FlatBinaryURL().String(), Equals, "http://repos.express42.com/virool/precise/Packages") +} + func (s *RemoteRepoSuite) TestPackageURL(c *C) { c.Assert(s.repo.PackageURL("pool/main/0/0ad/0ad_0~r11863-2_i386.deb").String(), Equals, "http://mirror.yandex.ru/debian/pool/main/0/0ad/0ad_0~r11863-2_i386.deb") @@ -215,6 +237,32 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { c.Check(pkg.Name, Equals, "amanda-client") } +func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { + downloader := utils.NewFakeDownloader() + downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile) + downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", errors.New("HTTP 404")) + downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", errors.New("HTTP 404")) + downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile) + downloader.ExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz") + + err := s.flat.Fetch(downloader, nil) + c.Assert(err, IsNil) + + err = s.flat.Download(downloader, s.packageCollection, s.packageRepo, false) + c.Assert(err, IsNil) + c.Assert(downloader.Empty(), Equals, true) + c.Assert(s.flat.packageRefs, NotNil) + + pkg, err := s.packageCollection.ByKey(s.flat.packageRefs.Refs[0]) + c.Assert(err, IsNil) + + result, err := pkg.VerifyFiles(s.packageRepo) + c.Check(result, Equals, true) + c.Check(err, IsNil) + + c.Check(pkg.Name, Equals, "amanda-client") +} + type RemoteRepoCollectionSuite struct { PackageListMixinSuite db database.Storage