Added max-tries flag for mirror update

aptly/interfaces.go

@@ -82,7 +82,7 @@ type Downloader interface {
 	// Download starts new download task
 	Download(url string, destination string, result chan<- error)
 	// DownloadWithChecksum starts new download task with checksum verification
-	DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool)
+	DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int)
 	// Pause pauses task processing
 	Pause()
 	// Resume resumes task processing

cmd/mirror_update.go

@@ -40,6 +40,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
+	maxTries := context.Flags().Lookup("max-tries").Value.Get().(int)
 
 	verifier, err := getVerifier(context.Flags())
 	if err != nil {
@@ -52,7 +53,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	context.Progress().Printf("Downloading & parsing package files...\n")
-	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch)
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch, maxTries)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -121,7 +122,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	// In separate goroutine (to avoid blocking main), push queue to downloader
 	go func() {
 		for _, task := range queue {
-			context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
+			context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch, maxTries)
 		}
 
 		// We don't need queue after this point
@@ -187,6 +188,7 @@ Example:
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
+	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

deb/remote.go

@@ -403,7 +403,7 @@ ok:
 
 // DownloadPackageIndexes downloads & parses package index files
 func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
-	ignoreMismatch bool) error {
+	ignoreMismatch bool, maxTries int) error {
 	if repo.packageList != nil {
 		panic("packageList != nil")
 	}
@@ -433,7 +433,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 
 	for _, info := range packagesURLs {
 		url, kind := info[0], info[1]
-		packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch)
+		packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch, maxTries)
 		if err != nil {
 			return err
 		}

http/download.go

@@ -52,6 +52,7 @@ type downloadTask struct {
 	result         chan<- error
 	expected       utils.ChecksumInfo
 	ignoreMismatch bool
+	triesLeft      int
 }
 
 // NewDownloader creates new instance of Downloader which specified number
@@ -127,13 +128,13 @@ func (downloader *downloaderImpl) GetProgress() aptly.Progress {
 
 // Download starts new download task
 func (downloader *downloaderImpl) Download(url string, destination string, result chan<- error) {
-	downloader.DownloadWithChecksum(url, destination, result, utils.ChecksumInfo{Size: -1}, false)
+	downloader.DownloadWithChecksum(url, destination, result, utils.ChecksumInfo{Size: -1}, false, 1)
 }
 
 // DownloadWithChecksum starts new download task with checksum verification
 func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, result chan<- error,
-	expected utils.ChecksumInfo, ignoreMismatch bool) {
+	expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) {
-	downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch}
+		downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch, triesLeft: maxTries}
 }
 
 // handleTask processes single download task
@@ -153,32 +154,59 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 		req.URL.RawQuery = ""
 	}
 
-	resp, err := downloader.client.Do(req)
+
+	var temppath string
+	for task.triesLeft > 0 {
+
+		temppath, err = downloader.downloadTask(req, task)
+
+		if err != nil {
+			task.triesLeft--
+		} else {
+			// successful download
+			break
+		}
+	}
+
+	// still an error after retring, giving up
 	if err != nil {
+		task.result <- err
+		return
+	}
+
+	err = os.Rename(temppath, task.destination)
+	if err != nil {
+		os.Remove(temppath)
 		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
 	}
+
+	task.result <- nil
+}
+
+func (downloader *downloaderImpl) downloadTask(req *http.Request, task *downloadTask) (string, error) {
+	resp, err := downloader.client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("%s: %s", task.url, err)
+	}
 	if resp.Body != nil {
 		defer resp.Body.Close()
 	}
 
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		task.result <- &HTTPError{Code: resp.StatusCode, URL: task.url}
+		return "", &HTTPError{Code: resp.StatusCode, URL: task.url}
-		return
 	}
 
 	err = os.MkdirAll(filepath.Dir(task.destination), 0777)
 	if err != nil {
-		task.result <- fmt.Errorf("%s: %s", task.url, err)
+		return "", fmt.Errorf("%s: %s", task.url, err)
-		return
 	}
 
 	temppath := task.destination + ".down"
 
 	outfile, err := os.Create(temppath)
 	if err != nil {
-		task.result <- fmt.Errorf("%s: %s", task.url, err)
+		return "", fmt.Errorf("%s: %s", task.url, err)
-		return
 	}
 	defer outfile.Close()
 
@@ -194,8 +222,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	_, err = io.Copy(w, resp.Body)
 	if err != nil {
 		os.Remove(temppath)
-		task.result <- fmt.Errorf("%s: %s", task.url, err)
+		return "", fmt.Errorf("%s: %s", task.url, err)
-		return
 	}
 
 	if task.expected.Size != -1 {
@@ -218,20 +245,12 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 				downloader.progress.Printf("WARNING: %s\n", err.Error())
 			} else {
 				os.Remove(temppath)
-				task.result <- err
+				return "", err
-				return
 			}
 		}
 	}
 
-	err = os.Rename(temppath, task.destination)
+	return temppath, nil
-	if err != nil {
-		os.Remove(temppath)
-		task.result <- fmt.Errorf("%s: %s", task.url, err)
-		return
-	}
-
-	task.result <- nil
 }
 
 // process implements download thread in goroutine
@@ -253,13 +272,13 @@ func (downloader *downloaderImpl) process() {
 //
 // Temporary file would be already removed, so no need to cleanup
 func DownloadTemp(downloader aptly.Downloader, url string) (*os.File, error) {
-	return DownloadTempWithChecksum(downloader, url, utils.ChecksumInfo{Size: -1}, false)
+	return DownloadTempWithChecksum(downloader, url, utils.ChecksumInfo{Size: -1}, false, 1)
 }
 
 // DownloadTempWithChecksum is a DownloadTemp with checksum verification
 //
 // Temporary file would be already removed, so no need to cleanup
-func DownloadTempWithChecksum(downloader aptly.Downloader, url string, expected utils.ChecksumInfo, ignoreMismatch bool) (*os.File, error) {
+func DownloadTempWithChecksum(downloader aptly.Downloader, url string, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (*os.File, error) {
 	tempdir, err := ioutil.TempDir(os.TempDir(), "aptly")
 	if err != nil {
 		return nil, err
@@ -274,7 +293,7 @@ func DownloadTempWithChecksum(downloader aptly.Downloader, url string, expected
 	}
 
 	ch := make(chan error, 1)
-	downloader.DownloadWithChecksum(url, tempfile, ch, expected, ignoreMismatch)
+	downloader.DownloadWithChecksum(url, tempfile, ch, expected, ignoreMismatch, maxTries)
 
 	err = <-ch
 
@@ -311,7 +330,7 @@ var compressionMethods = []struct {
 
 // DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until
 // it finds existing file.
-func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool) (io.Reader, *os.File, error) {
+func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (io.Reader, *os.File, error) {
 	var err error
 
 	for _, method := range compressionMethods {
@@ -322,7 +341,7 @@ func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChe
 
 		for suffix, expected := range expectedChecksums {
 			if strings.HasSuffix(tryURL, suffix) {
-				file, err = DownloadTempWithChecksum(downloader, tryURL, expected, ignoreMismatch)
+				file, err = DownloadTempWithChecksum(downloader, tryURL, expected, ignoreMismatch, maxTries)
 				foundChecksum = true
 				break
 			}

http/fake.go

@@ -59,7 +59,7 @@ func (f *FakeDownloader) Empty() bool {
 }
 
 // DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification
-func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool) {
+func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) {
 	var expectation expectedRequest
 	if len(f.expected) > 0 && f.expected[0].URL == url {
 		expectation, f.expected = f.expected[0], f.expected[1:]
@@ -116,7 +116,7 @@ func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, resul
 
 // Download performs fake download by matching against first expectation in the queue
 func (f *FakeDownloader) Download(url string, filename string, result chan<- error) {
-	f.DownloadWithChecksum(url, filename, result, utils.ChecksumInfo{Size: -1}, false)
+	f.DownloadWithChecksum(url, filename, result, utils.ChecksumInfo{Size: -1}, false, 1)
 }
 
 // Shutdown does nothing