Merge pull request #1411 from schoenherrg/feature/filter-using-file

Feature: Support Reading Filter Expressions from a File

cmd/mirror_create.go

@@ -46,7 +46,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
-	repo.Filter = context.Flags().Lookup("filter").Value.String()
+	repo.Filter = context.Flags().Lookup("filter").Value.String() // allows file/stdin with @
 	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
 	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
 	repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
@@ -103,7 +103,7 @@ Example:
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
-	cmd.Flag.String("filter", "", "filter packages in mirror")
+	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
 	cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")

cmd/mirror_edit.go

@@ -32,7 +32,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 	context.Flags().Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
-			repo.Filter = flag.Value.String()
+			repo.Filter = flag.Value.String() // allows file/stdin with @
 		case "filter-with-deps":
 			repo.FilterWithDeps = flag.Value.Get().(bool)
 		case "with-installer":
@@ -104,7 +104,7 @@ Example:
 	}
 
 	cmd.Flag.String("archive-url", "", "archive url is the root of archive")
-	cmd.Flag.String("filter", "", "filter packages in mirror")
+	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")

cmd/package_search.go

@@ -21,7 +21,11 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 	}
 
 	if len(args) == 1 {
-		q, err = query.Parse(args[0])
+		value, err := GetStringOrFileContent(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
+		}
+		q, err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -49,6 +53,7 @@ func makeCmdPackageSearch() *commander.Command {
 		Long: `
 Command search displays list of packages in whole DB that match package query.
 
+Use '@file' to read query from file or '@-' for stdin.
 If query is not specified, all the packages are displayed.
 
 Example:

cmd/package_show.go

@@ -66,7 +66,11 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	q, err := query.Parse(args[0])
+	value, err := GetStringOrFileContent(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
+	}
+	q, err := query.Parse(value)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -130,6 +134,8 @@ matching query. Information from Debian control file is displayed.
 Optionally information about package files and
 inclusion into mirrors/snapshots/local repos is shown.
 
+Use '@file' to read query from file or '@-' for stdin.
+
 Example:
 
     $ aptly package show 'nginx-light_1.2.1-2.2+wheezy2_i386'

cmd/repo_move.go

@@ -110,7 +110,11 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	queries := make([]deb.PackageQuery, len(args)-2)
 	for i := 0; i < len(args)-2; i++ {
-		queries[i], err = query.Parse(args[i+2])
+		value, err := GetStringOrFileContent(args[i+2])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[i+2], err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -186,6 +190,8 @@ func makeCmdRepoMove() *commander.Command {
 Command move moves packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
+Use '@file' to read package queries from file or '@-' for stdin.
+
 Example:
 
   $ aptly repo move testing stable 'myapp (=0.1.12)'

cmd/repo_remove.go

@@ -38,7 +38,11 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 
 	queries := make([]deb.PackageQuery, len(args)-1)
 	for i := 0; i < len(args)-1; i++ {
-		queries[i], err = query.Parse(args[i+1])
+		value, err := GetStringOrFileContent(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[i+1], err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to remove: %s", err)
 		}
@@ -81,6 +85,8 @@ Commands removes packages matching <package-query> from local repository
 snapshots, they can be removed completely (including files) by running
 'aptly db cleanup'.
 
+Use '@file' to read package queries from file or '@-' for stdin.
+
 Example:
 
   $ aptly repo remove testing 'myapp (=0.1.12)'

cmd/snapshot_filter.go

@@ -60,7 +60,11 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	// Initial queries out of arguments
 	queries := make([]deb.PackageQuery, len(args)-2)
 	for i, arg := range args[2:] {
-		queries[i], err = query.Parse(arg)
+		value, err := GetStringOrFileContent(arg)
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", arg, err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to parse query: %s", err)
 		}
@@ -103,6 +107,8 @@ Command filter does filtering in snapshot <source>, producing another
 snapshot <destination>. Packages could be specified simply
 as 'package-name' or as package queries.
 
+Use '@file' syntax to read package queries from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot filter wheezy-main wheezy-required 'Priority (required)'

cmd/snapshot_pull.go

@@ -88,7 +88,11 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	// Initial queries out of arguments
 	queries := make([]deb.PackageQuery, len(args)-3)
 	for i, arg := range args[3:] {
-		queries[i], err = query.Parse(arg)
+		value, err := GetStringOrFileContent(arg)
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", arg, err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to parse query: %s", err)
 		}
@@ -167,6 +171,8 @@ versions from <source> following dependencies. New snapshot <destination>
 is created as a result of this process. Packages could be specified simply
 as 'package-name' or as package queries.
 
+Use '@file' syntax to read package queries from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot pull wheezy-main wheezy-backports wheezy-new-xorg xorg-server-server

cmd/snapshot_search.go

@@ -78,7 +78,11 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 	list.PrepareIndex()
 
 	if len(args) == 2 {
-		q, err = query.Parse(args[1])
+		value, err := GetStringOrFileContent(args[1])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[1], err)
+		}
+		q, err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -134,6 +138,8 @@ Command search displays list of packages in snapshot that match package query
 
 If query is not specified, all the packages are displayed.
 
+Use '@file' syntax to read package query from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'

cmd/string_or_file_flag.go

@@ -0,0 +1,55 @@
+package cmd
+
+import (
+	"io"
+	"os"
+	"strings"
+
+	"github.com/smira/flag"
+)
+
+// StringOrFileFlag is a custom flag type that can handle both string input and file input.
+// If the input starts with '@', it is treated as a filename and the contents are read from the file.
+// If the input is '@-', the contents are read from stdin.
+type StringOrFileFlag struct {
+	value string
+}
+
+func (s *StringOrFileFlag) String() string {
+	return s.value
+}
+
+func (s *StringOrFileFlag) Set(value string) error {
+	var err error
+	s.value, err = GetStringOrFileContent(value)
+	return err
+}
+
+func (s *StringOrFileFlag) Get() any {
+	return s.value
+}
+
+func AddStringOrFileFlag(flagSet *flag.FlagSet, name string, value string, usage string) *StringOrFileFlag {
+	result := &StringOrFileFlag{value: value}
+	flagSet.Var(result, name, usage)
+	return result
+}
+
+func GetStringOrFileContent(value string) (string, error) {
+	if !strings.HasPrefix(value, "@") {
+		return value, nil
+	}
+
+	filename := strings.TrimPrefix(value, "@")
+	var data []byte
+	var err error
+	if filename == "-" { // Read from stdin
+		data, err = io.ReadAll(os.Stdin)
+	} else {
+		data, err = os.ReadFile(filename)
+	}
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}

man/aptly.1

@@ -588,7 +588,7 @@ Options:
 .
 .TP
 \-\fBfilter\fR=
-filter packages in mirror
+filter packages in mirror, use \(cq@file\(cq to read filter from file or \(cq@\-\(cq for stdin
 .
 .TP
 \-\fBfilter\-with\-deps\fR
@@ -771,7 +771,7 @@ archive url is the root of archive
 .
 .TP
 \-\fBfilter\fR=
-filter packages in mirror
+filter packages in mirror, use \(cq@file\(cq to read filter from file or \(cq@\-\(cq for stdin
 .
 .TP
 \-\fBfilter\-with\-deps\fR
@@ -1016,6 +1016,9 @@ display list in machine\-readable format
 Command move moves packages matching \fIpackage\-query\fR from local repo \fIsrc\-name\fR to local repo \fIdst\-name\fR\.
 .
 .P
+Use \(cq@file\(cq to read package queries from file or \(cq@\-\(cq for stdin\.
+.
+.P
 Example:
 .
 .P
@@ -1039,6 +1042,9 @@ follow dependencies when processing package\-spec
 Commands removes packages matching \fIpackage\-query\fR from local repository \fIname\fR\. If removed packages are not referenced by other repos or snapshots, they can be removed completely (including files) by running \(cqaptly db cleanup\(cq\.
 .
 .P
+Use \(cq@file\(cq to read package queries from file or \(cq@\-\(cq for stdin\.
+.
+.P
 Example:
 .
 .P
@@ -1263,6 +1269,9 @@ $ aptly snapshot verify wheezy\-main wheezy\-contrib wheezy\-non\-free
 Command pull pulls new packages along with its\(cq dependencies to snapshot \fIname\fR from snapshot \fIsource\fR\. Pull can upgrade package version in \fIname\fR with versions from \fIsource\fR following dependencies\. New snapshot \fIdestination\fR is created as a result of this process\. Packages could be specified simply as \(cqpackage\-name\(cq or as package queries\.
 .
 .P
+Use \(cq@file\(cq syntax to read package queries from file and \(cq@\-\(cq to read from stdin\.
+.
+.P
 Example:
 .
 .IP "" 4
@@ -1398,6 +1407,9 @@ Command search displays list of packages in snapshot that match package query
 If query is not specified, all the packages are displayed\.
 .
 .P
+Use \(cq@file\(cq syntax to read package query from file and \(cq@\-\(cq to read from stdin\.
+.
+.P
 Example:
 .
 .IP "" 4
@@ -1428,6 +1440,9 @@ include dependencies into search results
 Command filter does filtering in snapshot \fIsource\fR, producing another snapshot \fIdestination\fR\. Packages could be specified simply as \(cqpackage\-name\(cq or as package queries\.
 .
 .P
+Use \(cq@file\(cq syntax to read package queries from file and \(cq@\-\(cq to read from stdin\.
+.
+.P
 Example:
 .
 .IP "" 4
@@ -2206,7 +2221,7 @@ don\(cqt sign Release files with GPG
 Command search displays list of packages in whole DB that match package query\.
 .
 .P
-If query is not specified, all the packages are displayed\.
+Use \(cq@file\(cq to read query from file or \(cq@\-\(cq for stdin\. If query is not specified, all the packages are displayed\.
 .
 .P
 Example:
@@ -2235,6 +2250,9 @@ custom format for result printing
 Command shows displays detailed meta\-information about packages matching query\. Information from Debian control file is displayed\. Optionally information about package files and inclusion into mirrors/snapshots/local repos is shown\.
 .
 .P
+Use \(cq@file\(cq to read query from file or \(cq@\-\(cq for stdin\.
+.
+.P
 Example:
 .
 .IP "" 4
@@ -2653,5 +2671,8 @@ Blake Kostner (https://github\.com/btkostner)
 .IP "\[ci]" 4
 Leigh London (https://github\.com/leighlondon)
 .
+.IP "\[ci]" 4
+Gordian Schönherr (https://github\.com/schoenherrg)
+.
 .IP "" 0
 

system/t03_help/MirrorCreateHelpTest_gold

@@ -21,7 +21,7 @@ Options:
   -dep-follow-source: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests: when processing dependencies, follow Suggests
   -dep-verbose-resolve: when processing dependencies, print detailed logs
-  -filter="": filter packages in mirror
+  -filter=: filter packages in mirror, use '@file' to read filter from file or '@-' for stdin
   -filter-with-deps: when filtering, include dependencies of matching packages as well
   -force-architectures: (only with architecture list) skip check that requested architectures are listed in Release file
   -force-components: (only with component list) skip check that requested components are listed in Release file

system/t03_help/MirrorCreateTest_gold

@@ -12,7 +12,7 @@ Options:
   -dep-follow-source: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests: when processing dependencies, follow Suggests
   -dep-verbose-resolve: when processing dependencies, print detailed logs
-  -filter="": filter packages in mirror
+  -filter=: filter packages in mirror, use '@file' to read filter from file or '@-' for stdin
   -filter-with-deps: when filtering, include dependencies of matching packages as well
   -force-architectures: (only with architecture list) skip check that requested architectures are listed in Release file
   -force-components: (only with component list) skip check that requested components are listed in Release file

system/t03_help/WrongFlagTest_gold

@@ -13,7 +13,7 @@ Options:
   -dep-follow-source: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests: when processing dependencies, follow Suggests
   -dep-verbose-resolve: when processing dependencies, print detailed logs
-  -filter="": filter packages in mirror
+  -filter=: filter packages in mirror, use '@file' to read filter from file or '@-' for stdin
   -filter-with-deps: when filtering, include dependencies of matching packages as well
   -force-architectures: (only with architecture list) skip check that requested architectures are listed in Release file
   -force-components: (only with component list) skip check that requested components are listed in Release file

system/t04_mirror/CreateMirror36Test_gold

@@ -0,0 +1,4 @@
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release
+
+Mirror [mirror36]: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/ stretch/updates successfully added.
+You can run 'aptly mirror update mirror36' to download repository contents.

system/t04_mirror/CreateMirror36Test_mirror_show

@@ -0,0 +1,22 @@
+Name: mirror36
+Archive Root URL: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/
+Distribution: stretch/updates
+Components: main
+Architectures: amd64, arm64, armel, armhf, i386
+Download Sources: no
+Download .udebs: no
+Filter: nginx | Priority (required)
+Filter With Deps: no
+Last update: never
+
+Information from release file:
+Acquire-By-Hash: yes
+Architectures: amd64 arm64 armel armhf i386
+Codename: stretch
+Components: updates/main updates/contrib updates/non-free
+Description:  Debian 9 Security Updates
+
+Label: Debian-Security
+Origin: Debian
+Suite: oldoldstable
+Version: 9

system/t04_mirror/CreateMirror37Test_gold

@@ -0,0 +1,4 @@
+Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release
+
+Mirror [mirror37]: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/ stretch/updates successfully added.
+You can run 'aptly mirror update mirror37' to download repository contents.

system/t04_mirror/CreateMirror37Test_mirror_show

@@ -0,0 +1,22 @@
+Name: mirror37
+Archive Root URL: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/
+Distribution: stretch/updates
+Components: main
+Architectures: amd64, arm64, armel, armhf, i386
+Download Sources: no
+Download .udebs: no
+Filter: nginx | Priority (required)
+Filter With Deps: no
+Last update: never
+
+Information from release file:
+Acquire-By-Hash: yes
+Architectures: amd64 arm64 armel armhf i386
+Codename: stretch
+Components: updates/main updates/contrib updates/non-free
+Description:  Debian 9 Security Updates
+
+Label: Debian-Security
+Origin: Debian
+Suite: oldoldstable
+Version: 9

system/t04_mirror/create.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 import re
 import os
 
@@ -489,3 +490,45 @@ class CreateMirror35Test(BaseTest):
     def check(self):
         self.check_output()
         self.check_cmd_output("aptly mirror show mirror35", "mirror_show")
+
+
+class CreateMirror36Test(BaseTest):
+    """
+    create mirror: mirror with filter read from file
+    """
+    filterFilePath = os.path.join(os.environ["HOME"], ".aptly-filter.tmp")
+    fixtureCmds = [f"bash -c \"echo -n 'nginx | Priority (required)' > {filterFilePath}\""]
+    runCmd = f"aptly mirror create -ignore-signatures -filter='@{filterFilePath}' mirror36 http://repo.aptly.info/system-tests/archive.debian.org/debian-security/ stretch/updates main"
+
+    def check(self):
+        def removeDates(s):
+            return re.sub(r"(Date|Valid-Until): [,0-9:+A-Za-z -]+\n", "", s)
+
+        self.check_output()
+        self.check_cmd_output("aptly mirror show mirror36",
+                              "mirror_show", match_prepare=removeDates)
+
+
+class CreateMirror37Test(BaseTest):
+    """
+    create mirror: mirror with filter read from stdin
+    """
+    aptly_testing_bin = Path(__file__).parent.parent.parent / "aptly.test"
+    # Hack: Normally the test system detects if runCmd is an aptly command and then
+    # substitutes the aptly_testing_bin path and deletes the last three lines of output.
+    # However, I need to run it in bash to control stdin, so I have to do it manually.
+    runCmd = [
+        "bash",
+        "-c",
+        f"echo -n 'nginx | Priority (required)' | {aptly_testing_bin} mirror create " +
+        "-ignore-signatures -filter='@-' mirror37 http://repo.aptly.info/system-tests/archive.debian.org/debian-security/ stretch/updates main " +
+        "| grep -vE '^(EXIT|PASS|coverage:)'"
+    ]
+
+    def check(self):
+        def removeDates(s):
+            return re.sub(r"(Date|Valid-Until): [,0-9:+A-Za-z -]+\n", "", s)
+
+        self.check_output()
+        self.check_cmd_output("aptly mirror show mirror37",
+                              "mirror_show", match_prepare=removeDates)