feat: add AppStream (DEP-11) download support to RemoteRepo

deb/package_test.go

@@ -59,7 +59,7 @@ func (s *PackageSuite) TestNewUdebFromPara(c *C) {
 }
 
 func (s *PackageSuite) TestNewInstallerFromPara(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	downloader := http.NewFakeDownloader()
 	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs")
 	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list")

deb/publish_test.go

@@ -115,7 +115,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.reflist = NewPackageRefListFromPackageList(s.list)
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	repo.packageRefs = s.reflist
 	_ = s.factory.RemoteRepoCollection().Add(repo)
 

deb/remote.go

@@ -70,6 +70,10 @@ type RemoteRepo struct {
 	DownloadUdebs bool
 	// Should we download installer files?
 	DownloadInstaller bool
+	// Should we download AppStream (DEP-11) metadata?
+	DownloadAppStream bool
+	// AppStream files: relative path (e.g. "main/dep11/Components-amd64.yml.gz") → pool path
+	AppStreamFiles map[string]string `codec:"AppStreamFiles" json:"-"`
 	// Packages for json output
 	Packages []string `codec:"-" json:",omitempty"`
 	// "Snapshot" of current list of packages
@@ -82,7 +86,7 @@ type RemoteRepo struct {
 
 // NewRemoteRepo creates new instance of Debian remote repository with specified params
 func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string,
-	architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool) (*RemoteRepo, error) {
+	architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool, downloadAppStream bool) (*RemoteRepo, error) {
 	result := &RemoteRepo{
 		UUID:              uuid.NewString(),
 		Name:              name,
@@ -93,6 +97,7 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone
 		DownloadSources:   downloadSources,
 		DownloadUdebs:     downloadUdebs,
 		DownloadInstaller: downloadInstaller,
+		DownloadAppStream: downloadAppStream,
 	}
 
 	err := result.prepare()
@@ -147,6 +152,9 @@ func (repo *RemoteRepo) String() string {
 	if repo.DownloadInstaller {
 		srcFlag += " [installer]"
 	}
+	if repo.DownloadAppStream {
+		srcFlag += " [appstream]"
+	}
 	distribution := repo.Distribution
 	if distribution == "" {
 		distribution = "./"
@@ -264,6 +272,82 @@ func (repo *RemoteRepo) InstallerPath(component string, architecture string) str
 	return fmt.Sprintf("%s/installer-%s/current/images/SHA256SUMS", component, architecture)
 }
 
+// AppStreamPaths returns dep11 file paths from ReleaseFiles for a given component
+func (repo *RemoteRepo) AppStreamPaths(component string) []string {
+	prefix := component + "/dep11/"
+	var paths []string
+	for path := range repo.ReleaseFiles {
+		if strings.HasPrefix(path, prefix) {
+			paths = append(paths, path)
+		}
+	}
+	sort.Strings(paths)
+	return paths
+}
+
+// DownloadAppStreamFiles downloads AppStream (DEP-11) metadata files and imports them into the pool
+func (repo *RemoteRepo) DownloadAppStreamFiles(progress aptly.Progress, d aptly.Downloader,
+	packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage, ignoreChecksums bool) error {
+
+	repo.AppStreamFiles = make(map[string]string)
+
+	for _, component := range repo.Components {
+		paths := repo.AppStreamPaths(component)
+		if len(paths) == 0 {
+			continue
+		}
+
+		for _, relativePath := range paths {
+			info, ok := repo.ReleaseFiles[relativePath]
+			if !ok {
+				continue
+			}
+
+			url := repo.IndexesRootURL().ResolveReference(&url.URL{Path: relativePath}).String()
+
+			if progress != nil {
+				progress.Printf("Downloading AppStream file %s...\n", relativePath)
+			}
+
+			tempDir, err := os.MkdirTemp("", "aptly-appstream-*")
+			if err != nil {
+				return fmt.Errorf("unable to create temp dir for AppStream file %s: %s", relativePath, err)
+			}
+
+			tempPath := path.Join(tempDir, path.Base(relativePath))
+
+			var expected *utils.ChecksumInfo
+			if !ignoreChecksums {
+				expected = &info
+			}
+
+			err = d.DownloadWithChecksum(gocontext.TODO(), url, tempPath, expected, ignoreChecksums)
+			if err != nil {
+				_ = os.RemoveAll(tempDir)
+				// Skip files that are not found (some repos list dep11 files but don't serve them)
+				if herr, ok := err.(*http.Error); ok && (herr.Code == 404 || herr.Code == 403) {
+					if progress != nil {
+						progress.ColoredPrintf("@y[!]@| @!skipping AppStream file %s: not found@|", relativePath)
+					}
+					continue
+				}
+				return fmt.Errorf("unable to download AppStream file %s: %s", relativePath, err)
+			}
+
+			basename := path.Base(relativePath)
+			poolPath, err := packagePool.Import(tempPath, basename, &info, true, checksumStorage)
+			_ = os.RemoveAll(tempDir)
+			if err != nil {
+				return fmt.Errorf("unable to import AppStream file %s: %s", relativePath, err)
+			}
+
+			repo.AppStreamFiles[relativePath] = poolPath
+		}
+	}
+
+	return nil
+}
+
 // PackageURL returns URL of package file relative to repository root
 // architecture
 func (repo *RemoteRepo) PackageURL(filename string) *url.URL {

deb/remote_test.go

@@ -90,8 +90,8 @@ type RemoteRepoSuite struct {
 var _ = Suite(&RemoteRepoSuite{})
 
 func (s *RemoteRepoSuite) SetUpTest(c *C) {
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false, false)
-	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false)
+	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false, false)
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress(false)
 	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
@@ -108,7 +108,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
-	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false)
+	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	c.Assert(err, ErrorMatches, ".*(hexadecimal escape in host|percent-encoded characters in host|invalid URL escape).*")
 }
 
@@ -117,11 +117,11 @@ func (s *RemoteRepoSuite) TestFlatCreation(c *C) {
 	c.Check(s.flat.Distribution, Equals, "./")
 	c.Check(s.flat.Components, IsNil)
 
-	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false)
+	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false, false)
 	c.Check(flat2.IsFlat(), Equals, true)
 	c.Check(flat2.Distribution, Equals, "./binary/")
 
-	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false)
+	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false, false)
 	c.Check(err, ErrorMatches, "components aren't supported for flat repos")
 }
 
@@ -236,13 +236,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false, false)
 	err := s.repo.Fetch(s.downloader, nil, true)
 	c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*")
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false, false)
 	err := s.repo.Fetch(s.downloader, nil, true)
 	c.Assert(err, ErrorMatches, "component xyz not available in repo.*")
 }
@@ -706,7 +706,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 	_, err := s.collection.ByName("yandex")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
@@ -724,7 +724,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 	_, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 
 	r, err := s.collection.ByUUID(repo.UUID)
@@ -738,7 +738,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection := NewRemoteRepoCollection(s.db)
@@ -759,7 +759,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	_ = s.collection.Add(repo)
 
 	count := 0
@@ -781,10 +781,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestDrop(c *C) {
-	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	_ = s.collection.Add(repo1)
 
-	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false)
+	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false, false)
 	_ = s.collection.Add(repo2)
 
 	r1, _ := s.collection.ByUUID(repo1.UUID)

deb/snapshot_test.go

@@ -19,7 +19,7 @@ var _ = Suite(&SnapshotSuite{})
 
 func (s *SnapshotSuite) SetUpTest(c *C) {
 	s.SetUpPackages()
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	s.repo.packageRefs = s.reflist
 }
 
@@ -118,11 +118,11 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
 	s.collection = NewSnapshotCollection(s.db)
 	s.SetUpPackages()
 
-	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	s.repo1.packageRefs = s.reflist
 	s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1)
 
-	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false)
+	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false, false)
 	s.repo2.packageRefs = s.reflist
 	s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2)
 
@@ -223,7 +223,7 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) {
 	c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1})
 	c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2})
 
-	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false)
+	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false, false)
 
 	c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot(nil))
 }