From 1afcd68e019a69ea96dec86fd92cefed8ef28a4c Mon Sep 17 00:00:00 2001 From: Lorenzo Bolla Date: Fri, 28 Jan 2022 11:13:22 +0100 Subject: [PATCH] Make downloader type configurable --- cmd/mirror_update.go | 1 + completion.d/_aptly | 1 + completion.d/aptly | 2 +- context/context.go | 12 ++++++++++-- go.mod | 2 +- go.sum | 4 ++-- http/grab.go | 2 +- system/t02_config/ConfigShowTest_gold | 1 + system/t02_config/CreateConfigTest_gold | 1 + utils/config.go | 2 ++ utils/config_test.go | 1 + 11 files changed, 22 insertions(+), 7 deletions(-) diff --git a/cmd/mirror_update.go b/cmd/mirror_update.go index 614e61d3..111590bb 100644 --- a/cmd/mirror_update.go +++ b/cmd/mirror_update.go @@ -267,6 +267,7 @@ Example: cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror") cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)") + cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)") cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error") cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)") diff --git a/completion.d/_aptly b/completion.d/_aptly index d4cffc53..1ab08034 100644 --- a/completion.d/_aptly +++ b/completion.d/_aptly @@ -204,6 +204,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could update) _arguments \ "-download-limit=[limit download speed (kB/s)]:kB/s: " \ + "-downloader=[downloader to use]:str: " \ "-force=[force update mirror even if it is locked by another process]:$bool" \ "-ignore-checksums=[ignore checksum mismatches while downloading package files and metadata]:$bool" \ "-ignore-signatures=[disable verification of Release file signatures]:$bool" \ diff --git a/completion.d/aptly b/completion.d/aptly index bd1b64c3..6b7cc598 100644 --- a/completion.d/aptly +++ b/completion.d/aptly @@ -216,7 +216,7 @@ _aptly() "update") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-force -download-limit= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur})) + COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur})) else COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur})) fi diff --git a/context/context.go b/context/context.go index c732b0e4..55a09acd 100644 --- a/context/context.go +++ b/context/context.go @@ -227,8 +227,16 @@ func (context *AptlyContext) newDownloader(progress aptly.Progress) aptly.Downlo // If flag is defined prefer it to global setting maxTries = maxTriesFlag.Value.Get().(int) } - // return http.NewDownloader(downloadLimit*1024, maxTries, progress) - return http.NewGrabDownloader(downloadLimit*1024, maxTries, progress) + var downloader string = context.config().Downloader + downloaderFlag := context.flags.Lookup("downloader") + if downloaderFlag != nil { + downloader = downloaderFlag.Value.String() + } + + if downloader == "grab" { + return http.NewGrabDownloader(downloadLimit*1024, maxTries, progress) + } + return http.NewDownloader(downloadLimit*1024, maxTries, progress) } // Downloader returns instance of current downloader diff --git a/go.mod b/go.mod index 276b8eb0..81f0e500 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/DisposaBoy/JsonConfigReader v0.0.0-20130112093355-33a99fdf1d5e github.com/awalterschulze/gographviz v0.0.0-20160912181450-761fd5fbb34e github.com/aws/aws-sdk-go v1.25.0 - github.com/cavaliercoder/grab v2.0.0+incompatible + github.com/cavaliergopher/grab/v3 v3.0.1 github.com/cheggaaa/pb v1.0.10 github.com/fatih/color v1.7.0 // indirect github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7 // indirect diff --git a/go.sum b/go.sum index f2851f97..de0298a0 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,8 @@ github.com/awalterschulze/gographviz v0.0.0-20160912181450-761fd5fbb34e h1:24jix github.com/awalterschulze/gographviz v0.0.0-20160912181450-761fd5fbb34e/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs= github.com/aws/aws-sdk-go v1.25.0 h1:MyXUdCesJLBvSSKYcaKeeEwxNUwUpG6/uqVYeH/Zzfo= github.com/aws/aws-sdk-go v1.25.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/cavaliercoder/grab v2.0.0+incompatible h1:wZHbBQx56+Yxjx2TCGDcenhh3cJn7cCLMfkEPmySTSE= -github.com/cavaliercoder/grab v2.0.0+incompatible/go.mod h1:tTBkfNqSBfuMmMBFaO2phgyhdYhiZQ/+iXCZDzcDsMI= +github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4= +github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= github.com/cheggaaa/pb v1.0.10 h1:CNg2511WECXZ7Ja6jjyz9CMBpQOrMuP5+H5zfjgVi/Q= github.com/cheggaaa/pb v1.0.10/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= diff --git a/http/grab.go b/http/grab.go index 7218cd17..b284783b 100644 --- a/http/grab.go +++ b/http/grab.go @@ -14,7 +14,7 @@ import ( "golang.org/x/time/rate" "github.com/aptly-dev/aptly/utils" - "github.com/cavaliercoder/grab" + "github.com/cavaliergopher/grab/v3" "github.com/pkg/errors" "github.com/aptly-dev/aptly/aptly" diff --git a/system/t02_config/ConfigShowTest_gold b/system/t02_config/ConfigShowTest_gold index df79c7c8..860dd73c 100644 --- a/system/t02_config/ConfigShowTest_gold +++ b/system/t02_config/ConfigShowTest_gold @@ -3,6 +3,7 @@ "downloadConcurrency": 4, "downloadSpeedLimit": 0, "downloadRetries": 5, + "downloader": "default", "databaseOpenAttempts": 10, "architectures": [], "dependencyFollowSuggests": false, diff --git a/system/t02_config/CreateConfigTest_gold b/system/t02_config/CreateConfigTest_gold index 1abf281e..b13762cc 100644 --- a/system/t02_config/CreateConfigTest_gold +++ b/system/t02_config/CreateConfigTest_gold @@ -3,6 +3,7 @@ "downloadConcurrency": 4, "downloadSpeedLimit": 0, "downloadRetries": 0, + "downloader": "default", "databaseOpenAttempts": -1, "architectures": [], "dependencyFollowSuggests": false, diff --git a/utils/config.go b/utils/config.go index 0a0a7f88..1d247a29 100644 --- a/utils/config.go +++ b/utils/config.go @@ -12,6 +12,7 @@ type ConfigStructure struct { // nolint: maligned DownloadConcurrency int `json:"downloadConcurrency"` DownloadLimit int64 `json:"downloadSpeedLimit"` DownloadRetries int `json:"downloadRetries"` + Downloader string `json:"downloader"` DatabaseOpenAttempts int `json:"databaseOpenAttempts"` Architectures []string `json:"architectures"` DepFollowSuggests bool `json:"dependencyFollowSuggests"` @@ -87,6 +88,7 @@ var Config = ConfigStructure{ RootDir: filepath.Join(os.Getenv("HOME"), ".aptly"), DownloadConcurrency: 4, DownloadLimit: 0, + Downloader: "default", DatabaseOpenAttempts: -1, Architectures: []string{}, DepFollowSuggests: false, diff --git a/utils/config_test.go b/utils/config_test.go index 16c618cd..d96e24a2 100644 --- a/utils/config_test.go +++ b/utils/config_test.go @@ -63,6 +63,7 @@ func (s *ConfigSuite) TestSaveConfig(c *C) { " \"downloadConcurrency\": 5,\n"+ " \"downloadSpeedLimit\": 0,\n"+ " \"downloadRetries\": 0,\n"+ + " \"downloader\": \"\",\n"+ " \"databaseOpenAttempts\": 5,\n"+ " \"architectures\": null,\n"+ " \"dependencyFollowSuggests\": false,\n"+