diff --git a/deb/package_test.go b/deb/package_test.go index e2574fc8..165716b5 100644 --- a/deb/package_test.go +++ b/deb/package_test.go @@ -59,7 +59,7 @@ func (s *PackageSuite) TestNewUdebFromPara(c *C) { } func (s *PackageSuite) TestNewInstallerFromPara(c *C) { - repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false) + repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false, false) downloader := http.NewFakeDownloader() downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs") downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list") diff --git a/deb/publish_test.go b/deb/publish_test.go index 8037d994..a00b06c2 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -115,7 +115,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) { s.reflist = NewPackageRefListFromPackageList(s.list) - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) repo.packageRefs = s.reflist _ = s.factory.RemoteRepoCollection().Add(repo) diff --git a/deb/remote.go b/deb/remote.go index 8776b65a..aa4f1151 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -70,6 +70,10 @@ type RemoteRepo struct { DownloadUdebs bool // Should we download installer files? DownloadInstaller bool + // Should we download AppStream (DEP-11) metadata? + DownloadAppStream bool + // AppStream files: relative path (e.g. "main/dep11/Components-amd64.yml.gz") → pool path + AppStreamFiles map[string]string `codec:"AppStreamFiles" json:"-"` // Packages for json output Packages []string `codec:"-" json:",omitempty"` // "Snapshot" of current list of packages @@ -82,7 +86,7 @@ type RemoteRepo struct { // NewRemoteRepo creates new instance of Debian remote repository with specified params func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string, - architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool) (*RemoteRepo, error) { + architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool, downloadAppStream bool) (*RemoteRepo, error) { result := &RemoteRepo{ UUID: uuid.NewString(), Name: name, @@ -93,6 +97,7 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone DownloadSources: downloadSources, DownloadUdebs: downloadUdebs, DownloadInstaller: downloadInstaller, + DownloadAppStream: downloadAppStream, } err := result.prepare() @@ -147,6 +152,9 @@ func (repo *RemoteRepo) String() string { if repo.DownloadInstaller { srcFlag += " [installer]" } + if repo.DownloadAppStream { + srcFlag += " [appstream]" + } distribution := repo.Distribution if distribution == "" { distribution = "./" @@ -264,6 +272,82 @@ func (repo *RemoteRepo) InstallerPath(component string, architecture string) str return fmt.Sprintf("%s/installer-%s/current/images/SHA256SUMS", component, architecture) } +// AppStreamPaths returns dep11 file paths from ReleaseFiles for a given component +func (repo *RemoteRepo) AppStreamPaths(component string) []string { + prefix := component + "/dep11/" + var paths []string + for path := range repo.ReleaseFiles { + if strings.HasPrefix(path, prefix) { + paths = append(paths, path) + } + } + sort.Strings(paths) + return paths +} + +// DownloadAppStreamFiles downloads AppStream (DEP-11) metadata files and imports them into the pool +func (repo *RemoteRepo) DownloadAppStreamFiles(progress aptly.Progress, d aptly.Downloader, + packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage, ignoreChecksums bool) error { + + repo.AppStreamFiles = make(map[string]string) + + for _, component := range repo.Components { + paths := repo.AppStreamPaths(component) + if len(paths) == 0 { + continue + } + + for _, relativePath := range paths { + info, ok := repo.ReleaseFiles[relativePath] + if !ok { + continue + } + + url := repo.IndexesRootURL().ResolveReference(&url.URL{Path: relativePath}).String() + + if progress != nil { + progress.Printf("Downloading AppStream file %s...\n", relativePath) + } + + tempDir, err := os.MkdirTemp("", "aptly-appstream-*") + if err != nil { + return fmt.Errorf("unable to create temp dir for AppStream file %s: %s", relativePath, err) + } + + tempPath := path.Join(tempDir, path.Base(relativePath)) + + var expected *utils.ChecksumInfo + if !ignoreChecksums { + expected = &info + } + + err = d.DownloadWithChecksum(gocontext.TODO(), url, tempPath, expected, ignoreChecksums) + if err != nil { + _ = os.RemoveAll(tempDir) + // Skip files that are not found (some repos list dep11 files but don't serve them) + if herr, ok := err.(*http.Error); ok && (herr.Code == 404 || herr.Code == 403) { + if progress != nil { + progress.ColoredPrintf("@y[!]@| @!skipping AppStream file %s: not found@|", relativePath) + } + continue + } + return fmt.Errorf("unable to download AppStream file %s: %s", relativePath, err) + } + + basename := path.Base(relativePath) + poolPath, err := packagePool.Import(tempPath, basename, &info, true, checksumStorage) + _ = os.RemoveAll(tempDir) + if err != nil { + return fmt.Errorf("unable to import AppStream file %s: %s", relativePath, err) + } + + repo.AppStreamFiles[relativePath] = poolPath + } + } + + return nil +} + // PackageURL returns URL of package file relative to repository root // architecture func (repo *RemoteRepo) PackageURL(filename string) *url.URL { diff --git a/deb/remote_test.go b/deb/remote_test.go index 88168585..42998536 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -90,8 +90,8 @@ type RemoteRepoSuite struct { var _ = Suite(&RemoteRepoSuite{}) func (s *RemoteRepoSuite) SetUpTest(c *C) { - s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false) - s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false) + s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false, false) + s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false, false) s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile) s.progress = console.NewProgress(false) s.db, _ = goleveldb.NewOpenDB(c.MkDir()) @@ -108,7 +108,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) { } func (s *RemoteRepoSuite) TestInvalidURL(c *C) { - _, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false) + _, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false, false) c.Assert(err, ErrorMatches, ".*(hexadecimal escape in host|percent-encoded characters in host|invalid URL escape).*") } @@ -117,11 +117,11 @@ func (s *RemoteRepoSuite) TestFlatCreation(c *C) { c.Check(s.flat.Distribution, Equals, "./") c.Check(s.flat.Components, IsNil) - flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false) + flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false, false) c.Check(flat2.IsFlat(), Equals, true) c.Check(flat2.Distribution, Equals, "./binary/") - _, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false) + _, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false, false) c.Check(err, ErrorMatches, "components aren't supported for flat repos") } @@ -236,13 +236,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) { } func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) { - s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false) + s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false, false) err := s.repo.Fetch(s.downloader, nil, true) c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*") } func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) { - s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false) + s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false, false) err := s.repo.Fetch(s.downloader, nil, true) c.Assert(err, ErrorMatches, "component xyz not available in repo.*") } @@ -706,7 +706,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) { _, err := s.collection.ByName("yandex") c.Assert(err, ErrorMatches, "*.not found") - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) c.Assert(s.collection.Add(repo), IsNil) c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists") @@ -724,7 +724,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) { _, err := s.collection.ByUUID("some-uuid") c.Assert(err, ErrorMatches, "*.not found") - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) c.Assert(s.collection.Add(repo), IsNil) r, err := s.collection.ByUUID(repo.UUID) @@ -738,7 +738,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) { } func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) { - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) c.Assert(s.collection.Update(repo), IsNil) collection := NewRemoteRepoCollection(s.db) @@ -759,7 +759,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) { } func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) { - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) _ = s.collection.Add(repo) count := 0 @@ -781,10 +781,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) { } func (s *RemoteRepoCollectionSuite) TestDrop(c *C) { - repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) _ = s.collection.Add(repo1) - repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false) + repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false, false) _ = s.collection.Add(repo2) r1, _ := s.collection.ByUUID(repo1.UUID) diff --git a/deb/snapshot_test.go b/deb/snapshot_test.go index cbd5285c..3f2aa14f 100644 --- a/deb/snapshot_test.go +++ b/deb/snapshot_test.go @@ -19,7 +19,7 @@ var _ = Suite(&SnapshotSuite{}) func (s *SnapshotSuite) SetUpTest(c *C) { s.SetUpPackages() - s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) s.repo.packageRefs = s.reflist } @@ -118,11 +118,11 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) { s.collection = NewSnapshotCollection(s.db) s.SetUpPackages() - s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) + s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) s.repo1.packageRefs = s.reflist s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1) - s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false) + s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false, false) s.repo2.packageRefs = s.reflist s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2) @@ -223,7 +223,7 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) { c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1}) c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2}) - repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false) + repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false, false) c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot(nil)) }