Merge pull request #1543 from PhilipCramer/feat/appstream-mirror-support

Add appstream support

AUTHORS

@@ -83,3 +83,4 @@ List of contributors, in chronological order:
 * Tim Foerster (https://github.com/tonobo)
 * Zhang Xiao (https://github.com/xzhang1)
 * Tom Nguyen (https://github.com/lecafard)
+* Philip Cramer (https://github.com/PhilipCramer)

api/mirror.go

@@ -102,6 +102,8 @@ type mirrorCreateParams struct {
 	DownloadUdebs bool `                     json:"DownloadUdebs"`
 	// Set "true" to mirror installer files
 	DownloadInstaller bool `                 json:"DownloadInstaller"`
+	// Set "true" to mirror AppStream (DEP-11) metadata
+	DownloadAppStream bool `                 json:"DownloadAppStream"`
 	// Set "true" to include dependencies of matching packages when filtering
 	FilterWithDeps bool `                    json:"FilterWithDeps"`
 	// Set "true" to skip if the given components are in the Release file
@@ -153,7 +155,7 @@ func apiMirrorsCreate(c *gin.Context) {
 	}
 
 	repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures,
-		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
+		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller, b.DownloadAppStream)
 
 	if err != nil {
 		AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
@@ -573,6 +575,14 @@ func apiMirrorsUpdate(c *gin.Context) {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
+		if remote.DownloadAppStream && !remote.IsFlat() {
+			err = remote.DownloadAppStreamFiles(out, downloader,
+				context.PackagePool(), collectionFactory.ChecksumCollection(nil), b.IgnoreChecksums)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
 		if remote.Filter != "" {
 			var filterQuery deb.PackageQuery
 

api/mirror_test.go

@@ -26,6 +26,21 @@ func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
 	c.Check(response.Body.String(), Equals, "{\"error\":\"unable to drop: mirror with name does-not-exist not found\"}")
 }
 
+func (s *MirrorSuite) TestCreateMirrorFlatWithAppStream(c *C) {
+	body, err := json.Marshal(gin.H{
+		"Name":              "test-flat-appstream",
+		"ArchiveURL":        "http://example.com/repo/",
+		"Distribution":      "./",
+		"DownloadAppStream":  true,
+	})
+	c.Assert(err, IsNil)
+
+	response, err := s.HTTPRequest("POST", "/api/mirrors", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 400)
+	c.Check(response.Body.String(), Matches, ".*AppStream.*flat.*")
+}
+
 func (s *MirrorSuite) TestCreateMirror(c *C) {
 	c.ExpectFailure("Need to mock downloads")
 	body, err := json.Marshal(gin.H{

cmd/db_cleanup.go

@@ -26,6 +26,7 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
 
 	// collect information about references packages...
 	existingPackageRefs := deb.NewPackageRefList()
+	referencedAppStreamFiles := []string{}
 
 	// used only in verbose mode to report package use source
 	packageRefSources := map[string][]string{}
@@ -55,6 +56,10 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
 			}
 		}
 
+		for _, poolPath := range repo.AppStreamFiles {
+			referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath)
+		}
+
 		return nil
 	})
 	if err != nil {
@@ -118,6 +123,11 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
 				return nil
 			})
 		}
+
+		for _, poolPath := range snapshot.AppStreamFiles {
+			referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath)
+		}
+
 		return nil
 	})
 	if err != nil {
@@ -236,6 +246,7 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	referencedFiles = append(referencedFiles, referencedAppStreamFiles...)
 	sort.Strings(referencedFiles)
 	context.Progress().ShutdownBar()
 

cmd/mirror_create.go

@@ -20,6 +20,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
 	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
 	downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
+	downloadAppStream := context.Flags().Lookup("with-appstream").Value.Get().(bool)
 	ignoreSignatures := context.Config().GpgDisableVerify
 	if context.Flags().IsSet("ignore-signatures") {
 		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
@@ -41,7 +42,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	}
 
 	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
-		downloadSources, downloadUdebs, downloadInstaller)
+		downloadSources, downloadUdebs, downloadInstaller, downloadAppStream)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
@@ -100,6 +101,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata")
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")

cmd/mirror_edit.go

@@ -35,6 +35,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 			repo.Filter = flag.Value.String() // allows file/stdin with @
 		case "filter-with-deps":
 			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-appstream":
+			repo.DownloadAppStream = flag.Value.Get().(bool)
 		case "with-installer":
 			repo.DownloadInstaller = flag.Value.Get().(bool)
 		case "with-sources":
@@ -53,6 +55,10 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
 	}
 
+	if repo.IsFlat() && repo.DownloadAppStream {
+		return fmt.Errorf("unable to edit: flat mirrors don't support AppStream (DEP-11) metadata")
+	}
+
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
 		if err != nil {
@@ -107,6 +113,7 @@ Example:
 	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata")
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")

cmd/mirror_show.go

@@ -61,6 +61,11 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
 		downloadUdebs = Yes
 	}
 	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
+	downloadAppStream := No
+	if repo.DownloadAppStream {
+		downloadAppStream = Yes
+	}
+	fmt.Printf("Download AppStream: %s\n", downloadAppStream)
 	if repo.Filter != "" {
 		fmt.Printf("Filter: %s\n", repo.Filter)
 		filterWithDeps := No

cmd/mirror_update.go

@@ -64,6 +64,15 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
+	if repo.DownloadAppStream && !repo.IsFlat() {
+		context.Progress().Printf("Downloading AppStream metadata...\n")
+		err = repo.DownloadAppStreamFiles(context.Progress(), context.Downloader(),
+			context.PackagePool(), collectionFactory.ChecksumCollection(nil), ignoreChecksums)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
+
 	if repo.Filter != "" {
 		context.Progress().Printf("Applying filter...\n")
 		var filterQuery deb.PackageQuery

completion.d/_aptly

@@ -185,6 +185,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             $keyring \
                             "-with-sources=[download source packages in addition to binary packages]:$bool" \
                             "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
+                            "-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \
                             "(-)2:new mirror name: " ":archive url:_urls" ":distribution:($dists)" "*:components:_values -s ' ' components $components"
                         ;;
                     list)
@@ -224,6 +225,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             "-filter-with-deps=[when filtering, include dependencies of matching packages as well]:$bool" \
                             "-with-sources=[download source packages in addition to binary packages]:$bool" \
                             "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
+                            "-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \
                             "(-)2:mirror name:$mirrors"
                         ;;
                     search)

completion.d/aptly

@@ -203,7 +203,7 @@ _aptly()
           "create")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
+                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-appstream -with-installer -with-sources -with-udebs" -- ${cur}))
                 return 0
               fi
             fi
@@ -211,7 +211,7 @@ _aptly()
           "edit")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
+                COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-appstream -with-installer -with-sources -with-udebs" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
               fi

deb/package_test.go

@@ -59,7 +59,7 @@ func (s *PackageSuite) TestNewUdebFromPara(c *C) {
 }
 
 func (s *PackageSuite) TestNewInstallerFromPara(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	downloader := http.NewFakeDownloader()
 	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs")
 	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list")

deb/publish.go

@@ -1055,6 +1055,38 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			}
 		}
 
+		// Pass-through AppStream (DEP-11) files from snapshots
+		for component, item := range p.sourceItems {
+			if item.snapshot == nil || len(item.snapshot.AppStreamFiles) == 0 {
+				continue
+			}
+
+			prefix := component + "/"
+			for relPath, poolPath := range item.snapshot.AppStreamFiles {
+				if !strings.HasPrefix(relPath, prefix) {
+					continue
+				}
+				withinComponent := strings.TrimPrefix(relPath, prefix)
+
+				poolFile, err := packagePool.Open(poolPath)
+				if err != nil {
+					return fmt.Errorf("unable to open AppStream file from pool: %v", err)
+				}
+
+				bufWriter, err := indexes.SkelIndex(component, withinComponent).BufWriter()
+				if err != nil {
+					poolFile.Close()
+					return fmt.Errorf("unable to generate AppStream index: %v", err)
+				}
+
+				_, err = bufio.NewReader(poolFile).WriteTo(bufWriter)
+				poolFile.Close()
+				if err != nil {
+					return fmt.Errorf("unable to write AppStream file: %v", err)
+				}
+			}
+		}
+
 		udebs := []bool{false}
 		if hadUdebs {
 			udebs = append(udebs, true)

deb/publish_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/database/goleveldb"
 	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/ugorji/go/codec"
 
 	. "gopkg.in/check.v1"
@@ -115,7 +116,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.reflist = NewPackageRefListFromPackageList(s.list)
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	repo.packageRefs = s.reflist
 	_ = s.factory.RemoteRepoCollection().Add(repo)
 
@@ -425,6 +426,81 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	c.Assert(err, IsNil)
 }
 
+func (s *PublishedRepoSuite) TestPublishAppStream(c *C) {
+	// Components + icons
+	content1 := []byte("DEP-11 test content for Components-amd64.yml.gz")
+	tmpFile1 := filepath.Join(c.MkDir(), "Components-amd64.yml.gz")
+	c.Assert(os.WriteFile(tmpFile1, content1, 0644), IsNil)
+
+	checksums1 := utils.ChecksumInfo{Size: int64(len(content1))}
+	poolPath1, err := s.packagePool.Import(tmpFile1, "Components-amd64.yml.gz", &checksums1, false, s.cs)
+	c.Assert(err, IsNil)
+
+	content2 := []byte("DEP-11 icons tar data")
+	tmpFile2 := filepath.Join(c.MkDir(), "icons-48x48.tar.gz")
+	c.Assert(os.WriteFile(tmpFile2, content2, 0644), IsNil)
+
+	checksums2 := utils.ChecksumInfo{Size: int64(len(content2))}
+	poolPath2, err := s.packagePool.Import(tmpFile2, "icons-48x48.tar.gz", &checksums2, false, s.cs)
+	c.Assert(err, IsNil)
+
+	// Include contrib file that should be skipped
+	contribContent := []byte("DEP-11 contrib content")
+	tmpFile3 := filepath.Join(c.MkDir(), "Components-contrib.yml.gz")
+	c.Assert(os.WriteFile(tmpFile3, contribContent, 0644), IsNil)
+
+	checksums3 := utils.ChecksumInfo{Size: int64(len(contribContent))}
+	poolPath3, err := s.packagePool.Import(tmpFile3, "Components-contrib.yml.gz", &checksums3, false, s.cs)
+	c.Assert(err, IsNil)
+
+	s.snapshot.AppStreamFiles = map[string]string{
+		"main/dep11/Components-amd64.yml.gz":    poolPath1,
+		"main/dep11/icons-48x48.tar.gz":         poolPath2,
+		"contrib/dep11/Components-amd64.yml.gz": poolPath3,
+	}
+
+	err = s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "")
+	c.Assert(err, IsNil)
+
+	// Both main files should exist
+	appstreamPath1 := filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/dep11/Components-amd64.yml.gz")
+	c.Check(appstreamPath1, PathExists)
+	actual1, err := os.ReadFile(appstreamPath1)
+	c.Assert(err, IsNil)
+	c.Check(actual1, DeepEquals, content1)
+
+	appstreamPath2 := filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/dep11/icons-48x48.tar.gz")
+	c.Check(appstreamPath2, PathExists)
+	actual2, err := os.ReadFile(appstreamPath2)
+	c.Assert(err, IsNil)
+	c.Check(actual2, DeepEquals, content2)
+
+	// Contrib file should not appear
+	contribPath := filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/contrib/dep11/Components-amd64.yml.gz")
+	_, statErr := os.Stat(contribPath)
+	c.Check(os.IsNotExist(statErr), Equals, true)
+
+	// Release file should reference AppStream files
+	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
+	c.Assert(err, IsNil)
+	defer rf.Close()
+
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(st["MD5Sum"], Matches, "(?s).*main/dep11/Components-amd64\\.yml\\.gz.*")
+	c.Check(st["SHA256"], Matches, "(?s).*main/dep11/Components-amd64\\.yml\\.gz.*")
+
+	// Pool open error
+	s.snapshot.AppStreamFiles = map[string]string{
+		"main/dep11/Components-amd64.yml.gz": "nonexistent/pool/path",
+	}
+
+	err = s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "")
+	c.Assert(err, ErrorMatches, "unable to open AppStream file from pool.*")
+}
+
 func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
 	err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
 	c.Assert(err, IsNil)

deb/remote.go

@@ -70,6 +70,10 @@ type RemoteRepo struct {
 	DownloadUdebs bool
 	// Should we download installer files?
 	DownloadInstaller bool
+	// Should we download AppStream (DEP-11) metadata?
+	DownloadAppStream bool
+	// AppStream files: relative path (e.g. "main/dep11/Components-amd64.yml.gz") → pool path
+	AppStreamFiles map[string]string `codec:"AppStreamFiles" json:"-"`
 	// Packages for json output
 	Packages []string `codec:"-" json:",omitempty"`
 	// "Snapshot" of current list of packages
@@ -82,7 +86,7 @@ type RemoteRepo struct {
 
 // NewRemoteRepo creates new instance of Debian remote repository with specified params
 func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string,
-	architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool) (*RemoteRepo, error) {
+	architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool, downloadAppStream bool) (*RemoteRepo, error) {
 	result := &RemoteRepo{
 		UUID:              uuid.NewString(),
 		Name:              name,
@@ -93,6 +97,7 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone
 		DownloadSources:   downloadSources,
 		DownloadUdebs:     downloadUdebs,
 		DownloadInstaller: downloadInstaller,
+		DownloadAppStream: downloadAppStream,
 	}
 
 	err := result.prepare()
@@ -111,6 +116,9 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone
 		if result.DownloadUdebs {
 			return nil, fmt.Errorf("debian-installer udebs aren't supported for flat repos")
 		}
+		if result.DownloadAppStream {
+			return nil, fmt.Errorf("AppStream (DEP-11) metadata isn't supported for flat repos")
+		}
 		result.Components = nil
 	}
 
@@ -147,6 +155,9 @@ func (repo *RemoteRepo) String() string {
 	if repo.DownloadInstaller {
 		srcFlag += " [installer]"
 	}
+	if repo.DownloadAppStream {
+		srcFlag += " [appstream]"
+	}
 	distribution := repo.Distribution
 	if distribution == "" {
 		distribution = "./"
@@ -264,6 +275,82 @@ func (repo *RemoteRepo) InstallerPath(component string, architecture string) str
 	return fmt.Sprintf("%s/installer-%s/current/images/SHA256SUMS", component, architecture)
 }
 
+// AppStreamPaths returns dep11 file paths from ReleaseFiles for a given component
+func (repo *RemoteRepo) AppStreamPaths(component string) []string {
+	prefix := component + "/dep11/"
+	var paths []string
+	for path := range repo.ReleaseFiles {
+		if strings.HasPrefix(path, prefix) {
+			paths = append(paths, path)
+		}
+	}
+	sort.Strings(paths)
+	return paths
+}
+
+// DownloadAppStreamFiles downloads AppStream (DEP-11) metadata files and imports them into the pool
+func (repo *RemoteRepo) DownloadAppStreamFiles(progress aptly.Progress, d aptly.Downloader,
+	packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage, ignoreChecksums bool) error {
+
+	repo.AppStreamFiles = make(map[string]string)
+
+	for _, component := range repo.Components {
+		paths := repo.AppStreamPaths(component)
+		if len(paths) == 0 {
+			continue
+		}
+
+		for _, relativePath := range paths {
+			info, ok := repo.ReleaseFiles[relativePath]
+			if !ok {
+				continue
+			}
+
+			url := repo.IndexesRootURL().ResolveReference(&url.URL{Path: relativePath}).String()
+
+			if progress != nil {
+				progress.Printf("Downloading AppStream file %s...\n", relativePath)
+			}
+
+			tempDir, err := os.MkdirTemp("", "aptly-appstream-*")
+			if err != nil {
+				return fmt.Errorf("unable to create temp dir for AppStream file %s: %s", relativePath, err)
+			}
+
+			tempPath := path.Join(tempDir, path.Base(relativePath))
+
+			var expected *utils.ChecksumInfo
+			if !ignoreChecksums {
+				expected = &info
+			}
+
+			err = d.DownloadWithChecksum(gocontext.TODO(), url, tempPath, expected, ignoreChecksums)
+			if err != nil {
+				_ = os.RemoveAll(tempDir)
+				// Skip files that are not found (some repos list dep11 files but don't serve them)
+				if herr, ok := err.(*http.Error); ok && (herr.Code == 404 || herr.Code == 403) {
+					if progress != nil {
+						progress.ColoredPrintf("@y[!]@| @!skipping AppStream file %s: not found@|", relativePath)
+					}
+					continue
+				}
+				return fmt.Errorf("unable to download AppStream file %s: %s", relativePath, err)
+			}
+
+			basename := path.Base(relativePath)
+			poolPath, err := packagePool.Import(tempPath, basename, &info, true, checksumStorage)
+			_ = os.RemoveAll(tempDir)
+			if err != nil {
+				return fmt.Errorf("unable to import AppStream file %s: %s", relativePath, err)
+			}
+
+			repo.AppStreamFiles[relativePath] = poolPath
+		}
+	}
+
+	return nil
+}
+
 // PackageURL returns URL of package file relative to repository root
 // architecture
 func (repo *RemoteRepo) PackageURL(filename string) *url.URL {

deb/remote_test.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"os"
 	"sort"
@@ -90,8 +91,8 @@ type RemoteRepoSuite struct {
 var _ = Suite(&RemoteRepoSuite{})
 
 func (s *RemoteRepoSuite) SetUpTest(c *C) {
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false, false)
-	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false)
+	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false, false)
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress(false)
 	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
@@ -108,7 +109,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
-	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false)
+	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	c.Assert(err, ErrorMatches, ".*(hexadecimal escape in host|percent-encoded characters in host|invalid URL escape).*")
 }
 
@@ -117,12 +118,15 @@ func (s *RemoteRepoSuite) TestFlatCreation(c *C) {
 	c.Check(s.flat.Distribution, Equals, "./")
 	c.Check(s.flat.Components, IsNil)
 
-	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false)
+	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false, false)
 	c.Check(flat2.IsFlat(), Equals, true)
 	c.Check(flat2.Distribution, Equals, "./binary/")
 
-	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false)
+	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false, false)
 	c.Check(err, ErrorMatches, "components aren't supported for flat repos")
+
+	_, err = NewRemoteRepo("fl", "http://some.repo/", "./", []string{}, []string{}, false, false, false, true)
+	c.Check(err, ErrorMatches, "AppStream \\(DEP-11\\) metadata isn't supported for flat repos")
 }
 
 func (s *RemoteRepoSuite) TestString(c *C) {
@@ -135,6 +139,43 @@ func (s *RemoteRepoSuite) TestString(c *C) {
 	s.flat.DownloadSources = true
 	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb] [installer]")
 	c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./ [src]")
+
+	s.repo.DownloadAppStream = true
+	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb] [installer] [appstream]")
+
+	// AppStream is not supported for flat repos, so no flat test here
+}
+
+func (s *RemoteRepoSuite) TestAppStreamPaths(c *C) {
+	s.repo.ReleaseFiles = nil
+	c.Check(s.repo.AppStreamPaths("main"), DeepEquals, []string(nil))
+
+	s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{
+		"main/binary-amd64/Packages":              {Size: 100},
+		"main/dep11/Components-amd64.yml.gz":       {Size: 200},
+		"main/dep11/Components-i386.yml.gz":         {Size: 300},
+		"main/dep11/icons-48x48.tar.gz":             {Size: 400},
+		"contrib/dep11/Components-amd64.yml.gz":     {Size: 500},
+		"main/source/Sources":                       {Size: 600},
+	}
+
+	paths := s.repo.AppStreamPaths("main")
+	c.Check(paths, DeepEquals, []string{
+		"main/dep11/Components-amd64.yml.gz",
+		"main/dep11/Components-i386.yml.gz",
+		"main/dep11/icons-48x48.tar.gz",
+	})
+
+	paths = s.repo.AppStreamPaths("contrib")
+	c.Check(paths, DeepEquals, []string{
+		"contrib/dep11/Components-amd64.yml.gz",
+	})
+
+	paths = s.repo.AppStreamPaths("non-free")
+	c.Check(paths, DeepEquals, []string(nil))
+
+	s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{}
+	c.Check(s.repo.AppStreamPaths("main"), DeepEquals, []string(nil))
 }
 
 func (s *RemoteRepoSuite) TestNumPackages(c *C) {
@@ -236,13 +277,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false, false)
 	err := s.repo.Fetch(s.downloader, nil, true)
 	c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*")
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false, false)
 	err := s.repo.Fetch(s.downloader, nil, true)
 	c.Assert(err, ErrorMatches, "component xyz not available in repo.*")
 }
@@ -684,6 +725,74 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Assert(s.flat.packageRefs, NotNil)
 }
 
+func (s *RemoteRepoSuite) TestDownloadAppStreamFiles(c *C) {
+	// No dep11 entries
+	s.repo.Components = []string{"main"}
+	s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{
+		"main/binary-amd64/Packages": {Size: 100},
+		"main/source/Sources":        {Size: 200},
+	}
+
+	err := s.repo.DownloadAppStreamFiles(s.progress, s.downloader, s.packagePool, s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(s.repo.AppStreamFiles, HasLen, 0)
+
+	s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{
+		"main/dep11/Components-amd64.yml.gz": {Size: 16},
+		"main/dep11/icons-48x48.tar.gz":      {Size: 16},
+		"main/binary-amd64/Packages":          {Size: 100},
+	}
+
+	downloader := http.NewFakeDownloader()
+	downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/Components-amd64.yml.gz", "dep11-components")
+	downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/icons-48x48.tar.gz", "dep11-icons-data")
+
+	err = s.repo.DownloadAppStreamFiles(s.progress, downloader, s.packagePool, s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(s.repo.AppStreamFiles, HasLen, 2)
+	c.Check(s.repo.AppStreamFiles["main/dep11/Components-amd64.yml.gz"], Not(Equals), "")
+	c.Check(s.repo.AppStreamFiles["main/dep11/icons-48x48.tar.gz"], Not(Equals), "")
+
+	// 404 skipped
+	s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{
+		"main/dep11/Components-amd64.yml.gz": {Size: 16},
+		"main/dep11/icons-48x48.tar.gz":      {Size: 15},
+	}
+
+	downloader = http.NewFakeDownloader()
+	downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/Components-amd64.yml.gz", "dep11-components")
+	downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/icons-48x48.tar.gz", &http.Error{Code: 404, URL: "http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/icons-48x48.tar.gz"})
+
+	err = s.repo.DownloadAppStreamFiles(s.progress, downloader, s.packagePool, s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(s.repo.AppStreamFiles, HasLen, 1)
+	c.Check(s.repo.AppStreamFiles["main/dep11/Components-amd64.yml.gz"], Not(Equals), "")
+
+	// Generic download error propagated
+	s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{
+		"main/dep11/Components-amd64.yml.gz": {Size: 18},
+	}
+
+	downloader = http.NewFakeDownloader()
+	downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/Components-amd64.yml.gz", fmt.Errorf("connection refused"))
+
+	err = s.repo.DownloadAppStreamFiles(s.progress, downloader, s.packagePool, s.cs, false)
+	c.Assert(err, ErrorMatches, "unable to download AppStream file.*connection refused")
+
+	// Bypass checksum validation
+	s.repo.ReleaseFiles = map[string]utils.ChecksumInfo{
+		"main/dep11/Components-amd64.yml.gz": {Size: 999, MD5: "bad", SHA256: "bad"},
+	}
+
+	downloader = http.NewFakeDownloader()
+	downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/dep11/Components-amd64.yml.gz", "dep11-components")
+
+	err = s.repo.DownloadAppStreamFiles(s.progress, downloader, s.packagePool, s.cs, true)
+	c.Assert(err, IsNil)
+	c.Check(s.repo.AppStreamFiles, HasLen, 1)
+	c.Check(s.repo.AppStreamFiles["main/dep11/Components-amd64.yml.gz"], Not(Equals), "")
+}
+
 type RemoteRepoCollectionSuite struct {
 	PackageListMixinSuite
 	db         database.Storage
@@ -706,7 +815,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 	_, err := s.collection.ByName("yandex")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
@@ -724,7 +833,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 	_, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 
 	r, err := s.collection.ByUUID(repo.UUID)
@@ -738,7 +847,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection := NewRemoteRepoCollection(s.db)
@@ -759,7 +868,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	_ = s.collection.Add(repo)
 
 	count := 0
@@ -781,10 +890,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestDrop(c *C) {
-	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	_ = s.collection.Add(repo1)
 
-	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false)
+	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false, false)
 	_ = s.collection.Add(repo2)
 
 	r1, _ := s.collection.ByUUID(repo1.UUID)

deb/snapshot.go

@@ -40,6 +40,9 @@ type Snapshot struct {
 	NotAutomatic         string
 	ButAutomaticUpgrades string
 
+	// AppStream files: relative path → pool path (pass-through from mirror)
+	AppStreamFiles map[string]string `json:",omitempty"`
+
 	packageRefs *PackageRefList
 }
 
@@ -59,6 +62,7 @@ func NewSnapshotFromRepository(name string, repo *RemoteRepo) (*Snapshot, error)
 		Origin:               repo.Meta["Origin"],
 		NotAutomatic:         repo.Meta["NotAutomatic"],
 		ButAutomaticUpgrades: repo.Meta["ButAutomaticUpgrades"],
+		AppStreamFiles:       repo.AppStreamFiles,
 		packageRefs:          repo.packageRefs,
 	}, nil
 }
@@ -94,14 +98,28 @@ func NewSnapshotFromRefList(name string, sources []*Snapshot, list *PackageRefLi
 		sourceUUIDs[i] = sources[i].UUID
 	}
 
+	// Merge AppStreamFiles from all source snapshots
+	var mergedAppStream map[string]string
+	for _, source := range sources {
+		if len(source.AppStreamFiles) > 0 {
+			if mergedAppStream == nil {
+				mergedAppStream = make(map[string]string)
+			}
+			for k, v := range source.AppStreamFiles {
+				mergedAppStream[k] = v
+			}
+		}
+	}
+
 	return &Snapshot{
-		UUID:        uuid.NewString(),
+		UUID:           uuid.NewString(),
-		Name:        name,
+		Name:           name,
-		CreatedAt:   time.Now(),
+		CreatedAt:      time.Now(),
-		SourceKind:  "snapshot",
+		SourceKind:     "snapshot",
-		SourceIDs:   sourceUUIDs,
+		SourceIDs:      sourceUUIDs,
-		Description: description,
+		Description:    description,
-		packageRefs: list,
+		AppStreamFiles: mergedAppStream,
+		packageRefs:    list,
 	}
 }
 

deb/snapshot_test.go

@@ -19,7 +19,7 @@ var _ = Suite(&SnapshotSuite{})
 
 func (s *SnapshotSuite) SetUpTest(c *C) {
 	s.SetUpPackages()
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	s.repo.packageRefs = s.reflist
 }
 
@@ -101,6 +101,62 @@ func (s *SnapshotSuite) TestEncodeDecode(c *C) {
 	c.Assert(snapshot2.packageRefs, IsNil)
 }
 
+func (s *SnapshotSuite) TestSnapshotFromRepositoryAppStream(c *C) {
+	s.repo.AppStreamFiles = map[string]string{
+		"main/dep11/Components-amd64.yml.gz": "ab/cd/Components-amd64.yml.gz",
+		"main/dep11/icons-48x48.tar.gz":      "ef/gh/icons-48x48.tar.gz",
+	}
+	snapshot, err := NewSnapshotFromRepository("snap-as", s.repo)
+	c.Assert(err, IsNil)
+	c.Check(snapshot.AppStreamFiles, DeepEquals, s.repo.AppStreamFiles)
+
+	s.repo.AppStreamFiles = nil
+	snapshot2, err := NewSnapshotFromRepository("snap-no-as", s.repo)
+	c.Assert(err, IsNil)
+	c.Check(snapshot2.AppStreamFiles, IsNil)
+}
+
+func (s *SnapshotSuite) TestSnapshotFromRefListAppStreamMerge(c *C) {
+	snap1, _ := NewSnapshotFromRepository("snap1", s.repo)
+	snap1.AppStreamFiles = map[string]string{
+		"main/dep11/Components-amd64.yml.gz": "aa/bb/Components-amd64.yml.gz",
+		"main/dep11/icons-48x48.tar.gz":      "cc/dd/icons-48x48.tar.gz",
+	}
+
+	snap2, _ := NewSnapshotFromRepository("snap2", s.repo)
+	snap2.AppStreamFiles = map[string]string{
+		"contrib/dep11/Components-amd64.yml.gz": "ee/ff/Components-amd64.yml.gz",
+		"main/dep11/Components-amd64.yml.gz":     "xx/yy/Components-amd64.yml.gz",
+	}
+
+	merged := NewSnapshotFromRefList("merged", []*Snapshot{snap1, snap2}, s.reflist, "Merged")
+
+	c.Check(len(merged.AppStreamFiles), Equals, 3)
+	c.Check(merged.AppStreamFiles["main/dep11/icons-48x48.tar.gz"], Equals, "cc/dd/icons-48x48.tar.gz")
+	c.Check(merged.AppStreamFiles["contrib/dep11/Components-amd64.yml.gz"], Equals, "ee/ff/Components-amd64.yml.gz")
+	c.Check(merged.AppStreamFiles["main/dep11/Components-amd64.yml.gz"], Equals, "xx/yy/Components-amd64.yml.gz")
+
+	snap3, _ := NewSnapshotFromRepository("snap3", s.repo)
+	snap3.AppStreamFiles = nil
+	snap4, _ := NewSnapshotFromRepository("snap4", s.repo)
+	snap4.AppStreamFiles = nil
+	merged2 := NewSnapshotFromRefList("merged2", []*Snapshot{snap3, snap4}, s.reflist, "Merged2")
+	c.Check(merged2.AppStreamFiles, IsNil)
+}
+
+func (s *SnapshotSuite) TestEncodeDecodeAppStream(c *C) {
+	snapshot, _ := NewSnapshotFromRepository("snap-as-enc", s.repo)
+	snapshot.AppStreamFiles = map[string]string{
+		"main/dep11/Components-amd64.yml.gz": "ab/cd/Components-amd64.yml.gz",
+		"main/dep11/icons-48x48.tar.gz":      "ef/gh/icons-48x48.tar.gz",
+	}
+
+	decoded := &Snapshot{}
+	c.Assert(decoded.Decode(snapshot.Encode()), IsNil)
+	c.Check(decoded.Name, Equals, snapshot.Name)
+	c.Check(decoded.AppStreamFiles, DeepEquals, snapshot.AppStreamFiles)
+}
+
 type SnapshotCollectionSuite struct {
 	PackageListMixinSuite
 	db                   database.Storage
@@ -118,11 +174,11 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
 	s.collection = NewSnapshotCollection(s.db)
 	s.SetUpPackages()
 
-	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
+	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false)
 	s.repo1.packageRefs = s.reflist
 	s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1)
 
-	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false)
+	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false, false)
 	s.repo2.packageRefs = s.reflist
 	s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2)
 
@@ -223,7 +279,7 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) {
 	c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1})
 	c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2})
 
-	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false)
+	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false, false)
 
 	c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot(nil))
 }

man/aptly.1

@@ -615,6 +615,10 @@ gpg keyring to use when verifying Release file (could be specified multiple time
 max download tries till process fails with download error
 .
 .TP
+\-\fBwith\-appstream\fR
+download AppStream (DEP\-11) metadata
+.
+.TP
 \-\fBwith\-installer\fR
 download additional not packaged installer files
 .
@@ -790,6 +794,10 @@ disable verification of Release file signatures
 gpg keyring to use when verifying Release file (could be specified multiple times)
 .
 .TP
+\-\fBwith\-appstream\fR
+download AppStream (DEP\-11) metadata
+.
+.TP
 \-\fBwith\-installer\fR
 download additional not packaged installer files
 .

system/t03_help/MirrorCreateHelpTest_gold

@@ -29,6 +29,7 @@ Options:
   -ignore-signatures: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
   -max-tries=1: max download tries till process fails with download error
+  -with-appstream: download AppStream (DEP-11) metadata
   -with-installer: download additional not packaged installer files
   -with-sources: download source packages in addition to binary packages
   -with-udebs: download .udeb packages (Debian installer support)

system/t03_help/MirrorCreateTest_gold

@@ -20,6 +20,7 @@ Options:
   -ignore-signatures: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
   -max-tries=1: max download tries till process fails with download error
+  -with-appstream: download AppStream (DEP-11) metadata
   -with-installer: download additional not packaged installer files
   -with-sources: download source packages in addition to binary packages
   -with-udebs: download .udeb packages (Debian installer support)

system/t03_help/WrongFlagTest_gold

@@ -21,6 +21,7 @@ Options:
   -ignore-signatures: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
   -max-tries=1: max download tries till process fails with download error
+  -with-appstream: download AppStream (DEP-11) metadata
   -with-installer: download additional not packaged installer files
   -with-sources: download source packages in addition to binary packages
   -with-udebs: download .udeb packages (Debian installer support)

system/t04_mirror/CreateMirror11Test_mirror_show

@@ -5,6 +5,7 @@ Components: main, contrib, non-free
 Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror13Test_mirror_show

@@ -5,6 +5,7 @@ Components: main, contrib, non-free
 Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror14Test_mirror_show

@@ -5,6 +5,7 @@ Components:
 Architectures: all, amd64, i386
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror17Test_mirror_show

@@ -5,6 +5,7 @@ Components: main, contrib, non-free
 Architectures: i386
 Download Sources: yes
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror18Test_mirror_show

@@ -5,6 +5,7 @@ Components: main
 Architectures: amd64, armel, i386, powerpc
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror19Test_mirror_show

@@ -5,6 +5,7 @@ Components: main
 Architectures: i386
 Download Sources: yes
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror1Test_mirror_show

@@ -5,6 +5,7 @@ Components: main, contrib, non-free
 Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror21Test_mirror_show

@@ -5,6 +5,7 @@ Components:
 Architectures: all
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror22Test_mirror_show

@@ -5,6 +5,7 @@ Components: main
 Architectures: amd64, arm64, armel, armhf, i386
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Filter: nginx | Priority (required)
 Filter With Deps: no
 Last update: never

system/t04_mirror/CreateMirror25Test_mirror_show

@@ -5,6 +5,7 @@ Components: main, contrib, non-free
 Architectures: i386
 Download Sources: no
 Download .udebs: yes
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror27Test_mirror_show

@@ -5,6 +5,7 @@ Components: openmanage/740
 Architectures: amd64, i386
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror2Test_mirror_show

@@ -5,6 +5,7 @@ Components: main
 Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror32Test_mirror_show

@@ -5,6 +5,7 @@ Components: main, contrib, non-free
 Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror33Test_mirror_show

@@ -5,6 +5,7 @@ Components:
 Architectures: amd64
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror35Test_mirror_show

@@ -5,6 +5,7 @@ Components:
 Architectures: amd64
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Filter: cuda-12-6 (= 12.6.2-1)
 Filter With Deps: yes
 Last update: never

system/t04_mirror/CreateMirror36Test_mirror_show

@@ -5,6 +5,7 @@ Components: main
 Architectures: amd64, arm64, armel, armhf, i386
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Filter: nginx | Priority (required)
 Filter With Deps: no
 Last update: never

system/t04_mirror/CreateMirror37Test_mirror_show

@@ -5,6 +5,7 @@ Components: main
 Architectures: amd64, arm64, armel, armhf, i386
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Filter: nginx | Priority (required)
 Filter With Deps: no
 Last update: never

system/t04_mirror/CreateMirror38Test_gold

@@ -0,0 +1,4 @@
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
+
+Mirror [mirror38]: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/ stretch [appstream] successfully added.
+You can run 'aptly mirror update mirror38' to download repository contents.

system/t04_mirror/CreateMirror38Test_mirror_show

@@ -0,0 +1,23 @@
+Name: mirror38
+Archive Root URL: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/
+Distribution: stretch
+Components: main, contrib, non-free
+Architectures: amd64
+Download Sources: no
+Download .udebs: no
+Download AppStream: yes
+Last update: never
+
+Information from release file:
+Acquire-By-Hash: yes
+Architectures: amd64 arm64 armel armhf i386 mips mips64el mipsel ppc64el s390x
+Changelogs: http://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog
+Codename: stretch
+Components: main contrib non-free
+Date: Sat, 14 Aug 2021 07:42:00 UTC
+Description:  Debian 9.13 Released 18 July 2020
+
+Label: Debian
+Origin: Debian
+Suite: oldoldstable
+Version: 9.13

system/t04_mirror/CreateMirror3Test_mirror_show

@@ -5,6 +5,7 @@ Components: main, contrib
 Architectures: i386, amd64
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror7Test_mirror_show

@@ -5,6 +5,7 @@ Components: main, contrib
 Architectures: i386, amd64
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror9Test_mirror_show

@@ -5,6 +5,7 @@ Components: main, contrib, non-free
 Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/EditMirror11Test_gold

@@ -0,0 +1 @@
+Mirror [wheezy-main]: http://mirror.yandex.ru/debian/ wheezy [appstream] successfully updated.

system/t04_mirror/EditMirror11Test_mirror_show

@@ -0,0 +1,21 @@
+Name: wheezy-main
+Archive Root URL: http://mirror.yandex.ru/debian/
+Distribution: wheezy
+Components: main
+Architectures: i386, amd64
+Download Sources: no
+Download .udebs: no
+Download AppStream: yes
+Number of packages: 56121
+
+Information from release file:
+Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
+Codename: wheezy
+Components: main contrib non-free
+Date: Sat, 26 Apr 2014 09:27:11 UTC
+Description:  Debian 7.5 Released 26 April 2014
+
+Label: Debian
+Origin: Debian
+Suite: stable
+Version: 7.5

system/t04_mirror/EditMirror1Test_mirror_show

@@ -5,6 +5,7 @@ Components: main
 Architectures: i386, amd64
 Download Sources: yes
 Download .udebs: no
+Download AppStream: no
 Filter: nginx
 Filter With Deps: yes
 Number of packages: 56121

system/t04_mirror/EditMirror3Test_mirror_show

@@ -5,6 +5,7 @@ Components: main
 Architectures: i386, amd64
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Number of packages: 56121
 
 Information from release file:

system/t04_mirror/EditMirror5Test_mirror_show

@@ -5,6 +5,7 @@ Components: main
 Architectures: amd64, arm64, armel, armhf, i386
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/EditMirror6Test_mirror_show

@@ -5,6 +5,7 @@ Components: main
 Architectures: amd64, i386
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 
 Information from release file:
 Acquire-By-Hash: yes

system/t04_mirror/EditMirror8Test_mirror_show

@@ -5,6 +5,7 @@ Components: main
 Architectures: i386, amd64
 Download Sources: no
 Download .udebs: yes
+Download AppStream: no
 Number of packages: 56121
 
 Information from release file:

system/t04_mirror/ListMirror6Test_gold

@@ -42,7 +42,8 @@
     "SkipArchitectureCheck": false,
     "DownloadSources": false,
     "DownloadUdebs": false,
-    "DownloadInstaller": false
+    "DownloadInstaller": false,
+    "DownloadAppStream": false
   },
   {
     "Name": "mirror2",
@@ -85,7 +86,8 @@
     "SkipArchitectureCheck": false,
     "DownloadSources": true,
     "DownloadUdebs": false,
-    "DownloadInstaller": false
+    "DownloadInstaller": false,
+    "DownloadAppStream": false
   },
   {
     "Name": "mirror3",
@@ -119,7 +121,8 @@
     "SkipArchitectureCheck": false,
     "DownloadSources": false,
     "DownloadUdebs": false,
-    "DownloadInstaller": false
+    "DownloadInstaller": false,
+    "DownloadAppStream": false
   },
   {
     "Name": "mirror4",
@@ -148,6 +151,7 @@
     "SkipArchitectureCheck": false,
     "DownloadSources": false,
     "DownloadUdebs": false,
-    "DownloadInstaller": false
+    "DownloadInstaller": false,
+    "DownloadAppStream": false
   }
 ]

system/t04_mirror/ShowMirror1Test_gold

@@ -5,6 +5,7 @@ Components: main, contrib, non-free
 Architectures: amd64, arm64, armel, armhf, i386, mips, mips64el, mipsel, ppc64el, s390x
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Last update: never
 
 Information from release file:

system/t04_mirror/ShowMirror3Test_gold

@@ -5,6 +5,7 @@ Components: contrib
 Architectures: i386, amd64
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Number of packages: 325
 
 Information from release file:

system/t04_mirror/ShowMirror4Test_gold

@@ -5,6 +5,7 @@ Components: main
 Architectures: amd64, arm64, armel, armhf, i386
 Download Sources: no
 Download .udebs: no
+Download AppStream: no
 Filter: nginx | Priority (required)
 Filter With Deps: yes
 Last update: never

system/t04_mirror/ShowMirror5Test_gold

@@ -41,5 +41,6 @@
   "SkipArchitectureCheck": false,
   "DownloadSources": false,
   "DownloadUdebs": false,
-  "DownloadInstaller": false
+  "DownloadInstaller": false,
+  "DownloadAppStream": false
 }

system/t04_mirror/ShowMirror7Test_gold

@@ -31,6 +31,7 @@
   "DownloadSources": false,
   "DownloadUdebs": false,
   "DownloadInstaller": false,
+  "DownloadAppStream": false,
   "Packages": [
     "alien-arena-server_7.53+dfsg-3_amd64",
     "alien-arena-server_7.53+dfsg-3_i386",

system/t04_mirror/ShowMirror8Test_gold

@@ -33,5 +33,6 @@
   "SkipArchitectureCheck": false,
   "DownloadSources": false,
   "DownloadUdebs": false,
-  "DownloadInstaller": false
+  "DownloadInstaller": false,
+  "DownloadAppStream": false
 }

system/t04_mirror/UpdateMirror28Test_gold

@@ -0,0 +1,26 @@
+Downloading: ${url}dists/hardy/Release
+Downloading & parsing package files...
+Downloading: ${url}dists/hardy/main/binary-amd64/Packages.bz2
+Error (retrying): HTTP code 404 while fetching ${url}dists/hardy/main/binary-amd64/Packages.bz2
+Retrying 0 ${url}dists/hardy/main/binary-amd64/Packages.bz2...
+Download Error: ${url}dists/hardy/main/binary-amd64/Packages.bz2
+Downloading: ${url}dists/hardy/main/binary-amd64/Packages.gz
+Error (retrying): HTTP code 404 while fetching ${url}dists/hardy/main/binary-amd64/Packages.gz
+Retrying 0 ${url}dists/hardy/main/binary-amd64/Packages.gz...
+Download Error: ${url}dists/hardy/main/binary-amd64/Packages.gz
+Downloading: ${url}dists/hardy/main/binary-amd64/Packages.xz
+Error (retrying): HTTP code 404 while fetching ${url}dists/hardy/main/binary-amd64/Packages.xz
+Retrying 0 ${url}dists/hardy/main/binary-amd64/Packages.xz...
+Download Error: ${url}dists/hardy/main/binary-amd64/Packages.xz
+Downloading: ${url}dists/hardy/main/binary-amd64/Packages
+Downloading AppStream metadata...
+Downloading AppStream file main/dep11/Components-amd64.yml.gz...
+Downloading: ${url}dists/hardy/main/dep11/Components-amd64.yml.gz
+Downloading AppStream file main/dep11/icons-48x48.tar.gz...
+Downloading: ${url}dists/hardy/main/dep11/icons-48x48.tar.gz
+Building download queue...
+Download queue: 1 items (30 B)
+Downloading: ${url}pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb
+WARNING: ${url}pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb: sha1 hash mismatch "8d3a014000038725d6daf8771b42a0784253688f" != "66b27417d37e024c46526c2f6d358a754fc552f3"
+
+Mirror `appstream-test` has been updated successfully.

system/t04_mirror/UpdateMirror28Test_mirror_show

@@ -0,0 +1,21 @@
+Name: appstream-test
+Archive Root URL: ${url}
+Distribution: hardy
+Components: main
+Architectures: amd64
+Download Sources: no
+Download .udebs: no
+Download AppStream: yes
+Number of packages: 1
+
+Information from release file:
+Architectures: amd64
+Codename: hardy
+Components: main
+Date: Sat, 19 Oct 2013 13:54:21 UTC
+Description:  Debian 6.0.8 Released 19 October 2013
+
+Label: failure
+Origin: test
+Suite: test
+Version: 6.0.8

system/t04_mirror/create.py

@@ -533,3 +533,14 @@ class CreateMirror37Test(BaseTest):
         self.check_output()
         self.check_cmd_output("aptly mirror show mirror37",
                               "mirror_show", match_prepare=removeDates)
+
+
+class CreateMirror38Test(BaseTest):
+    """
+    create mirror: mirror with appstream enabled
+    """
+    runCmd = "aptly -architectures=amd64 mirror create -ignore-signatures -with-appstream mirror38 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/ stretch"
+
+    def check(self):
+        self.check_output()
+        self.check_cmd_output("aptly mirror show mirror38", "mirror_show")

system/t04_mirror/edit.py

@@ -114,3 +114,16 @@ class EditMirror10Test(BaseTest):
     """
     fixtureCmds = ["aptly mirror create -ignore-signatures mirror10 http://repo.aptly.info/system-tests/ftp.ru.debian.org/debian bookworm main"]
     runCmd = "aptly mirror edit -ignore-signatures -archive-url http://repo.aptly.info/system-tests/ftp.ch.debian.org/debian mirror10"
+
+
+class EditMirror11Test(BaseTest):
+    """
+    edit mirror: enable appstream
+    """
+    fixtureDB = True
+    runCmd = "aptly mirror edit -with-appstream wheezy-main"
+
+    def check(self):
+        self.check_output()
+        self.check_cmd_output("aptly mirror show wheezy-main", "mirror_show",
+                              match_prepare=lambda s: re.sub(r"Last update: [0-9:+A-Za-z -]+\n", "", s))

system/t04_mirror/test_release2/dists/hardy/Release

@@ -9,3 +9,11 @@ Components: main
 Description: Debian 6.0.8 Released 19 October 2013
 MD5Sum:
  846549680001f5c632b6ee8e0f183825        827 main/binary-amd64/Packages
+ f72bdc4905a2d68a7a4b85544a7135c3         92 main/dep11/Components-amd64.yml.gz
+ 26dce1ff9274a848f1633737936044ed        170 main/dep11/icons-48x48.tar.gz
+SHA1:
+ 2a4d420955106ebf1765153341ba4581680f62ce         92 main/dep11/Components-amd64.yml.gz
+ 2f14d886f1e798b899184e7ff1ef2d2f27cd59df        170 main/dep11/icons-48x48.tar.gz
+SHA256:
+ 1245da5002d7e96c3b3133671e9267f69eea41ee4e3f3b8a0dcb6df15e5b51b7         92 main/dep11/Components-amd64.yml.gz
+ bdd3f75f27c2eaf78e074142ee2270809d5f16b26cc64c60df7cb38ae0f38516        170 main/dep11/icons-48x48.tar.gz

system/t04_mirror/update.py

@@ -482,3 +482,23 @@ class UpdateMirror27Test(BaseTest):
     runCmd = "aptly mirror update -downloader=grab -keyring=aptlytest.gpg grab-fail"
     outputMatchPrepare = filterOutRedirects
     expectedCode = 1
+
+
+class UpdateMirror28Test(BaseTest):
+    """
+    update mirrors: update with appstream
+    """
+    fixtureCmds = [
+        "aptly mirror create --ignore-signatures -with-appstream appstream-test ${url} hardy main",
+    ]
+    fixtureWebServer = "test_release2"
+    configOverride = {"downloadRetries": 0}
+    runCmd = "aptly mirror update -ignore-checksums --ignore-signatures appstream-test"
+
+    def gold_processor(self, gold):
+        return string.Template(gold).substitute({'url': self.webServerUrl})
+
+    def check(self):
+        self.check_output()
+        self.check_cmd_output("aptly mirror show appstream-test", "mirror_show",
+                              match_prepare=lambda s: re.sub(r"Last update: [0-9:+A-Za-z -]+\n", "", s))

system/t05_snapshot/ShowSnapshot4Test_gold

@@ -34,7 +34,8 @@
       "SkipArchitectureCheck": false,
       "DownloadSources": false,
       "DownloadUdebs": false,
-      "DownloadInstaller": false
+      "DownloadInstaller": false,
+      "DownloadAppStream": false
     }
   ],
   "Description": "Snapshot from mirror [wheezy-non-free]: http://mirror.yandex.ru/debian/ wheezy",

system/t05_snapshot/ShowSnapshot5Test_gold

@@ -36,7 +36,8 @@
       "SkipArchitectureCheck": false,
       "DownloadSources": false,
       "DownloadUdebs": false,
-      "DownloadInstaller": false
+      "DownloadInstaller": false,
+      "DownloadAppStream": false
     }
   ],
   "Packages": [

system/t06_publish/PublishSnapshot44Test_gold

@@ -0,0 +1,14 @@
+Loading packages...
+Generating metadata files and linking package files...
+[!] Failed to generate package contents:  unable to read .deb archive from amanda-client_3.3.1-3~bpo60+1_amd64.deb: ar: missing global header
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+
+Snapshot snap-appstream has been successfully published.
+Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing.
+Now you can add following line to apt sources:
+  deb http://your-server/ hardy main
+Don't forget to add your GPG key to apt with apt-key.
+
+You can also use `aptly serve` to publish your repositories over HTTP quickly.

system/t06_publish/snapshot.py

@@ -1439,3 +1439,24 @@ class PublishSnapshot43Test(BaseTest):
 
         self.check_file_contents(
             'public/dists/maverick/Release', 'release', match_prepare=strip_processor)
+
+
+class PublishSnapshot44Test(BaseTest):
+    """
+    publish snapshot: mirror with appstream
+    """
+    fixtureWebServer = "../t04_mirror/test_release2"
+    fixtureGpg = True
+    fixtureCmds = [
+        "aptly mirror create --ignore-signatures -with-appstream -architectures=amd64 appstream-test ${url} hardy main",
+        "aptly mirror update -ignore-checksums --ignore-signatures appstream-test",
+        "aptly snapshot create snap-appstream from mirror appstream-test",
+    ]
+    runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap-appstream"
+    gold_processor = BaseTest.expand_environ
+    configOverride = {"downloadRetries": 0}
+
+    def check(self):
+        super(PublishSnapshot44Test, self).check()
+        self.check_exists('public/dists/hardy/main/dep11/Components-amd64.yml.gz')
+        self.check_exists('public/dists/hardy/main/dep11/icons-48x48.tar.gz')

system/t08_db/CleanupDB13Test_gold

@@ -0,0 +1,7 @@
+Loading mirrors, local repos, snapshots and published repos...
+Loading list of all packages...
+Deleting unreferenced packages (0)...
+Building list of files referenced by packages...
+Building list of files in package pool...
+Deleting unreferenced files (0)...
+Compacting database...

system/t08_db/CleanupDB13Test_publish

@@ -0,0 +1,14 @@
+Loading packages...
+Generating metadata files and linking package files...
+[!] Failed to generate package contents:  unable to read .deb archive from amanda-client_3.3.1-3~bpo60+1_amd64.deb: ar: missing global header
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+
+Snapshot snap-appstream has been successfully published.
+Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing.
+Now you can add following line to apt sources:
+  deb http://your-server/ hardy main
+Don't forget to add your GPG key to apt with apt-key.
+
+You can also use `aptly serve` to publish your repositories over HTTP quickly.

system/t08_db/cleanup.py

@@ -150,3 +150,27 @@ class CleanupDB12Test(BaseTest):
         "aptly mirror drop gnuplot-maverick",
     ]
     runCmd = "aptly db cleanup -verbose -dry-run"
+
+
+class CleanupDB13Test(BaseTest):
+    """
+    cleanup db: appstream files survive cleanup
+    """
+    fixtureWebServer = "../t04_mirror/test_release2"
+    fixtureGpg = True
+    configOverride = {"downloadRetries": 0}
+    fixtureCmds = [
+        "aptly mirror create --ignore-signatures -with-appstream -architectures=amd64 appstream-test ${url} hardy main",
+        "aptly mirror update -ignore-checksums --ignore-signatures appstream-test",
+        "aptly snapshot create snap-appstream from mirror appstream-test",
+    ]
+    runCmd = "aptly db cleanup"
+
+    def check(self):
+        self.check_output()
+        # verify appstream files survive cleanup by publishing the snapshot
+        self.check_cmd_output(
+            "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap-appstream",
+            "publish", match_prepare=self.expand_environ)
+        self.check_exists('public/dists/hardy/main/dep11/Components-amd64.yml.gz')
+        self.check_exists('public/dists/hardy/main/dep11/icons-48x48.tar.gz')