Make downloader type configurable

cmd/mirror_update.go

@@ -267,6 +267,7 @@ Example:
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
+	cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
 	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 

completion.d/_aptly

@@ -204,6 +204,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                     update)
                         _arguments \
                             "-download-limit=[limit download speed (kB/s)]:kB/s: " \
+                            "-downloader=[downloader to use]:str: " \
                             "-force=[force update mirror even if it is locked by another process]:$bool" \
                             "-ignore-checksums=[ignore checksum mismatches while downloading package files and metadata]:$bool" \
                             "-ignore-signatures=[disable verification of Release file signatures]:$bool" \

completion.d/aptly

@@ -216,7 +216,7 @@ _aptly()
           "update")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-force -download-limit= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur}))
+                COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
               fi

context/context.go

@@ -227,8 +227,16 @@ func (context *AptlyContext) newDownloader(progress aptly.Progress) aptly.Downlo
 		// If flag is defined prefer it to global setting
 		maxTries = maxTriesFlag.Value.Get().(int)
 	}
-	// return http.NewDownloader(downloadLimit*1024, maxTries, progress)
+	var downloader string = context.config().Downloader
-	return http.NewGrabDownloader(downloadLimit*1024, maxTries, progress)
+	downloaderFlag := context.flags.Lookup("downloader")
+	if downloaderFlag != nil {
+		downloader = downloaderFlag.Value.String()
+	}
+
+	if downloader == "grab" {
+		return http.NewGrabDownloader(downloadLimit*1024, maxTries, progress)
+	}
+	return http.NewDownloader(downloadLimit*1024, maxTries, progress)
 }
 
 // Downloader returns instance of current downloader

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/DisposaBoy/JsonConfigReader v0.0.0-20130112093355-33a99fdf1d5e
 	github.com/awalterschulze/gographviz v0.0.0-20160912181450-761fd5fbb34e
 	github.com/aws/aws-sdk-go v1.25.0
-	github.com/cavaliercoder/grab v2.0.0+incompatible
+	github.com/cavaliergopher/grab/v3 v3.0.1
 	github.com/cheggaaa/pb v1.0.10
 	github.com/fatih/color v1.7.0 // indirect
 	github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7 // indirect

go.sum

@@ -20,8 +20,8 @@ github.com/awalterschulze/gographviz v0.0.0-20160912181450-761fd5fbb34e h1:24jix
 github.com/awalterschulze/gographviz v0.0.0-20160912181450-761fd5fbb34e/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs=
 github.com/aws/aws-sdk-go v1.25.0 h1:MyXUdCesJLBvSSKYcaKeeEwxNUwUpG6/uqVYeH/Zzfo=
 github.com/aws/aws-sdk-go v1.25.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/cavaliercoder/grab v2.0.0+incompatible h1:wZHbBQx56+Yxjx2TCGDcenhh3cJn7cCLMfkEPmySTSE=
+github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4=
-github.com/cavaliercoder/grab v2.0.0+incompatible/go.mod h1:tTBkfNqSBfuMmMBFaO2phgyhdYhiZQ/+iXCZDzcDsMI=
+github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4=
 github.com/cheggaaa/pb v1.0.10 h1:CNg2511WECXZ7Ja6jjyz9CMBpQOrMuP5+H5zfjgVi/Q=
 github.com/cheggaaa/pb v1.0.10/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=

http/grab.go

@@ -14,7 +14,7 @@ import (
 	"golang.org/x/time/rate"
 
 	"github.com/aptly-dev/aptly/utils"
-	"github.com/cavaliercoder/grab"
+	"github.com/cavaliergopher/grab/v3"
 	"github.com/pkg/errors"
 
 	"github.com/aptly-dev/aptly/aptly"

system/t02_config/ConfigShowTest_gold

@@ -3,6 +3,7 @@
     "downloadConcurrency": 4,
     "downloadSpeedLimit": 0,
     "downloadRetries": 5,
+    "downloader": "default",
     "databaseOpenAttempts": 10,
     "architectures": [],
     "dependencyFollowSuggests": false,

system/t02_config/CreateConfigTest_gold

@@ -3,6 +3,7 @@
   "downloadConcurrency": 4,
   "downloadSpeedLimit": 0,
   "downloadRetries": 0,
+  "downloader": "default",
   "databaseOpenAttempts": -1,
   "architectures": [],
   "dependencyFollowSuggests": false,

utils/config.go

@@ -12,6 +12,7 @@ type ConfigStructure struct { // nolint: maligned
 	DownloadConcurrency    int                              `json:"downloadConcurrency"`
 	DownloadLimit          int64                            `json:"downloadSpeedLimit"`
 	DownloadRetries        int                              `json:"downloadRetries"`
+	Downloader             string                           `json:"downloader"`
 	DatabaseOpenAttempts   int                              `json:"databaseOpenAttempts"`
 	Architectures          []string                         `json:"architectures"`
 	DepFollowSuggests      bool                             `json:"dependencyFollowSuggests"`
@@ -87,6 +88,7 @@ var Config = ConfigStructure{
 	RootDir:                filepath.Join(os.Getenv("HOME"), ".aptly"),
 	DownloadConcurrency:    4,
 	DownloadLimit:          0,
+	Downloader:             "default",
 	DatabaseOpenAttempts:   -1,
 	Architectures:          []string{},
 	DepFollowSuggests:      false,

utils/config_test.go

@@ -63,6 +63,7 @@ func (s *ConfigSuite) TestSaveConfig(c *C) {
 		"  \"downloadConcurrency\": 5,\n"+
 		"  \"downloadSpeedLimit\": 0,\n"+
 		"  \"downloadRetries\": 0,\n"+
+		"  \"downloader\": \"\",\n"+
 		"  \"databaseOpenAttempts\": 5,\n"+
 		"  \"architectures\": null,\n"+
 		"  \"dependencyFollowSuggests\": false,\n"+