Refactor query reading from file, update docs

Add support for @file syntax in more places.

cmd/cmd.go

@@ -7,9 +7,6 @@ import (
 	"os"
 	"text/template"
 	"time"
-	"bufio"
-	"io/ioutil"
-	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
@@ -132,35 +129,3 @@ package environment to new version.`,
 	}
 	return cmd
 }
-
-// Reads the content of a file. If the file is "-", reads from stdin.
-func getContent(filterarg string) (string, error) {
-	var err error
-	// Check if filterarg starts with '@'
-	if strings.HasPrefix(filterarg, "@") {
-		// Remove the '@' character from filterarg
-		filterarg = strings.TrimPrefix(filterarg, "@")
-        if filterarg == "-" {
-			// If filterarg is "-", read from stdin
-			scanner := bufio.NewScanner(os.Stdin)
-			scanner.Split(bufio.ScanLines)
-			scanner.Buffer(make([]byte, 1024*1024), 1024*1024)
-			var content strings.Builder
-			for scanner.Scan() {
-				content.WriteString(scanner.Text() + "\n")
-			}
-			err = scanner.Err()
-			if err == nil {
-				filterarg = content.String()
-			}
-		} else {
-			// Read the file content into a byte slice
-			var data []byte
-			data, err = ioutil.ReadFile(filterarg)
-			if err == nil {
-				filterarg = string(data)
-			}
-		}
-	}
-	return filterarg, err
-}

cmd/mirror_create.go

@@ -46,7 +46,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
-	repo.Filter, err = getContent(context.Flags().Lookup("filter").Value.String())
+	repo.Filter = context.Flags().Lookup("filter").Value.String()
 	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
 	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
 	repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
@@ -106,7 +106,7 @@ Example:
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
-	cmd.Flag.String("filter", "", "filter packages in mirror")
+	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
 	cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")

cmd/mirror_edit.go

@@ -29,12 +29,10 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 
 	fetchMirror := false
 	ignoreSignatures := context.Config().GpgDisableVerify
-	var f string
 	context.Flags().Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
-			repo.Filter, err = getContent(flag.Value.String())
+			repo.Filter = flag.Value.String()
-			f = flag.Value.String()
 		case "filter-with-deps":
 			repo.FilterWithDeps = flag.Value.Get().(bool)
 		case "with-installer":
@@ -51,10 +49,6 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		}
 	})
 
-	if repo.Filter != "" && err != nil {
-		return fmt.Errorf("unable to read package query from file %s: %w", f, err)
-	}
-
 	if repo.IsFlat() && repo.DownloadUdebs {
 		return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
 	}
@@ -110,7 +104,7 @@ Example:
 	}
 
 	cmd.Flag.String("archive-url", "", "archive url is the root of archive")
-	cmd.Flag.String("filter", "", "filter packages in mirror")
+	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")

cmd/package_search.go

@@ -21,7 +21,11 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 	}
 
 	if len(args) == 1 {
-		q, err = query.Parse(args[0])
+		value, err := GetStringOrFileContent(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
+		}
+		q, err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -49,6 +53,7 @@ func makeCmdPackageSearch() *commander.Command {
 		Long: `
 Command search displays list of packages in whole DB that match package query.
 
+Use '@file' to read query from file or '@-' for stdin.
 If query is not specified, all the packages are displayed.
 
 Example:

cmd/package_show.go

@@ -66,7 +66,11 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	q, err := query.Parse(args[0])
+	value, err := GetStringOrFileContent(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
+	}
+	q, err := query.Parse(value)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -130,6 +134,8 @@ matching query. Information from Debian control file is displayed.
 Optionally information about package files and
 inclusion into mirrors/snapshots/local repos is shown.
 
+Use '@file' to read query from file or '@-' for stdin.
+
 Example:
 
     $ aptly package show 'nginx-light_1.2.1-2.2+wheezy2_i386'

cmd/repo_move.go

@@ -110,7 +110,11 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	queries := make([]deb.PackageQuery, len(args)-2)
 	for i := 0; i < len(args)-2; i++ {
-		queries[i], err = query.Parse(args[i+2])
+		value, err := GetStringOrFileContent(args[i+2])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[i+2], err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -186,6 +190,8 @@ func makeCmdRepoMove() *commander.Command {
 Command move moves packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
+Use '@file' to read package queries from file or '@-' for stdin.
+
 Example:
 
   $ aptly repo move testing stable 'myapp (=0.1.12)'

cmd/repo_remove.go

@@ -38,7 +38,11 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 
 	queries := make([]deb.PackageQuery, len(args)-1)
 	for i := 0; i < len(args)-1; i++ {
-		queries[i], err = query.Parse(args[i+1])
+		value, err := GetStringOrFileContent(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[i+1], err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to remove: %s", err)
 		}
@@ -81,6 +85,8 @@ Commands removes packages matching <package-query> from local repository
 snapshots, they can be removed completely (including files) by running
 'aptly db cleanup'.
 
+Use '@file' to read package queries from file or '@-' for stdin.
+
 Example:
 
   $ aptly repo remove testing 'myapp (=0.1.12)'

cmd/snapshot_filter.go

@@ -60,12 +60,11 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	// Initial queries out of arguments
 	queries := make([]deb.PackageQuery, len(args)-2)
 	for i, arg := range args[2:] {
-		var q string
+		value, err := GetStringOrFileContent(arg)
-		q, err = getContent(arg)
 		if err != nil {
 			return fmt.Errorf("unable to read package query from file %s: %w", arg, err)
 		}
-		queries[i], err = query.Parse(q)
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to parse query: %s", err)
 		}
@@ -108,6 +107,8 @@ Command filter does filtering in snapshot <source>, producing another
 snapshot <destination>. Packages could be specified simply
 as 'package-name' or as package queries.
 
+Use '@file' syntax to read package queries from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot filter wheezy-main wheezy-required 'Priority (required)'

cmd/snapshot_pull.go

@@ -88,12 +88,11 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	// Initial queries out of arguments
 	queries := make([]deb.PackageQuery, len(args)-3)
 	for i, arg := range args[3:] {
-		var q string
+		value, err := GetStringOrFileContent(arg)
-		q, err = getContent(arg)
 		if err != nil {
 			return fmt.Errorf("unable to read package query from file %s: %w", arg, err)
 		}
-		queries[i], err = query.Parse(q)
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to parse query: %s", err)
 		}
@@ -172,6 +171,8 @@ versions from <source> following dependencies. New snapshot <destination>
 is created as a result of this process. Packages could be specified simply
 as 'package-name' or as package queries.
 
+Use '@file' syntax to read package queries from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot pull wheezy-main wheezy-backports wheezy-new-xorg xorg-server-server

cmd/snapshot_search.go

@@ -78,7 +78,11 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 	list.PrepareIndex()
 
 	if len(args) == 2 {
-		q, err = query.Parse(args[1])
+		value, err := GetStringOrFileContent(args[1])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[1], err)
+		}
+		q, err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -134,6 +138,8 @@ Command search displays list of packages in snapshot that match package query
 
 If query is not specified, all the packages are displayed.
 
+Use '@file' syntax to read package query from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'

cmd/string_or_file_flag.go

@@ -0,0 +1,55 @@
+package cmd
+
+import (
+	"io"
+	"os"
+	"strings"
+
+	"github.com/smira/flag"
+)
+
+// StringOrFileFlag is a custom flag type that can handle both string input and file input.
+// If the input starts with '@', it is treated as a filename and the contents are read from the file.
+// If the input is '@-', the contents are read from stdin.
+type StringOrFileFlag struct {
+	value string
+}
+
+func (s *StringOrFileFlag) String() string {
+	return s.value
+}
+
+func (s *StringOrFileFlag) Set(value string) error {
+	var err error
+	s.value, err = GetStringOrFileContent(value)
+	return err
+}
+
+func (s *StringOrFileFlag) Get() any {
+	return s.value
+}
+
+func AddStringOrFileFlag(flagSet *flag.FlagSet, name string, value string, usage string) *StringOrFileFlag {
+	result := &StringOrFileFlag{value: value}
+	flagSet.Var(result, name, usage)
+	return result
+}
+
+func GetStringOrFileContent(value string) (string, error) {
+	if !strings.HasPrefix(value, "@") {
+		return value, nil
+	}
+
+	filename := strings.TrimPrefix(value, "@")
+	var data []byte
+	var err error
+	if filename == "-" { // Read from stdin
+		data, err = io.ReadAll(os.Stdin)
+	} else {
+		data, err = os.ReadFile(filename)
+	}
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}