diff --git a/api/mirror.go b/api/mirror.go index 38b36b11..aa4af0a2 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -102,6 +102,8 @@ type mirrorCreateParams struct { DownloadUdebs bool ` json:"DownloadUdebs"` // Set "true" to mirror installer files DownloadInstaller bool ` json:"DownloadInstaller"` + // Set "true" to mirror AppStream (DEP-11) metadata + DownloadAppStream bool ` json:"DownloadAppStream"` // Set "true" to include dependencies of matching packages when filtering FilterWithDeps bool ` json:"FilterWithDeps"` // Set "true" to skip if the given components are in the Release file @@ -153,7 +155,7 @@ func apiMirrorsCreate(c *gin.Context) { } repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures, - b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller) + b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller, b.DownloadAppStream) if err != nil { AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err)) @@ -573,6 +575,14 @@ func apiMirrorsUpdate(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } + if remote.DownloadAppStream && !remote.IsFlat() { + err = remote.DownloadAppStreamFiles(out, downloader, + context.PackagePool(), collectionFactory.ChecksumCollection(nil), b.IgnoreChecksums) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + } + if remote.Filter != "" { var filterQuery deb.PackageQuery diff --git a/cmd/mirror_create.go b/cmd/mirror_create.go index 049dd2eb..d8a910d7 100644 --- a/cmd/mirror_create.go +++ b/cmd/mirror_create.go @@ -20,6 +20,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error { downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources") downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool) downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool) + downloadAppStream := context.Flags().Lookup("with-appstream").Value.Get().(bool) ignoreSignatures := context.Config().GpgDisableVerify if context.Flags().IsSet("ignore-signatures") { ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool) @@ -41,7 +42,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error { } repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(), - downloadSources, downloadUdebs, downloadInstaller) + downloadSources, downloadUdebs, downloadInstaller, downloadAppStream) if err != nil { return fmt.Errorf("unable to create mirror: %s", err) } @@ -100,6 +101,7 @@ Example: } cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") + cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata") cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files") cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages") cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)") diff --git a/cmd/mirror_edit.go b/cmd/mirror_edit.go index f95d9736..6e07d066 100644 --- a/cmd/mirror_edit.go +++ b/cmd/mirror_edit.go @@ -35,6 +35,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error { repo.Filter = flag.Value.String() // allows file/stdin with @ case "filter-with-deps": repo.FilterWithDeps = flag.Value.Get().(bool) + case "with-appstream": + repo.DownloadAppStream = flag.Value.Get().(bool) case "with-installer": repo.DownloadInstaller = flag.Value.Get().(bool) case "with-sources": @@ -107,6 +109,7 @@ Example: AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin") cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well") cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") + cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata") cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files") cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages") cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)") diff --git a/cmd/mirror_show.go b/cmd/mirror_show.go index 00018fb8..b106c656 100644 --- a/cmd/mirror_show.go +++ b/cmd/mirror_show.go @@ -61,6 +61,11 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error { downloadUdebs = Yes } fmt.Printf("Download .udebs: %s\n", downloadUdebs) + downloadAppStream := No + if repo.DownloadAppStream { + downloadAppStream = Yes + } + fmt.Printf("Download AppStream: %s\n", downloadAppStream) if repo.Filter != "" { fmt.Printf("Filter: %s\n", repo.Filter) filterWithDeps := No diff --git a/cmd/mirror_update.go b/cmd/mirror_update.go index df28efa7..d67fce3e 100644 --- a/cmd/mirror_update.go +++ b/cmd/mirror_update.go @@ -64,6 +64,15 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { return fmt.Errorf("unable to update: %s", err) } + if repo.DownloadAppStream && !repo.IsFlat() { + context.Progress().Printf("Downloading AppStream metadata...\n") + err = repo.DownloadAppStreamFiles(context.Progress(), context.Downloader(), + context.PackagePool(), collectionFactory.ChecksumCollection(nil), ignoreChecksums) + if err != nil { + return fmt.Errorf("unable to update: %s", err) + } + } + if repo.Filter != "" { context.Progress().Printf("Applying filter...\n") var filterQuery deb.PackageQuery diff --git a/deb/publish.go b/deb/publish.go index 9fda0707..69ffb545 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -1055,6 +1055,38 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } } + // Pass-through AppStream (DEP-11) files from snapshots + for component, item := range p.sourceItems { + if item.snapshot == nil || len(item.snapshot.AppStreamFiles) == 0 { + continue + } + + prefix := component + "/" + for relPath, poolPath := range item.snapshot.AppStreamFiles { + if !strings.HasPrefix(relPath, prefix) { + continue + } + withinComponent := strings.TrimPrefix(relPath, prefix) + + poolFile, err := packagePool.Open(poolPath) + if err != nil { + return fmt.Errorf("unable to open AppStream file from pool: %v", err) + } + + bufWriter, err := indexes.SkelIndex(component, withinComponent).BufWriter() + if err != nil { + poolFile.Close() + return fmt.Errorf("unable to generate AppStream index: %v", err) + } + + _, err = bufio.NewReader(poolFile).WriteTo(bufWriter) + poolFile.Close() + if err != nil { + return fmt.Errorf("unable to write AppStream file: %v", err) + } + } + } + udebs := []bool{false} if hadUdebs { udebs = append(udebs, true)