vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-01-12 03:21:33 +00:00
Code Issues Projects Releases Wiki Activity

Added max-tries flag for mirror update

Browse Source
This commit is contained in:
Oliver Sauder
2016-11-28 13:13:11 +01:00
parent 653255c728
commit 6becd5a3aa
5 changed files with 55 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
aptly/interfaces.go
View File

@@ -82,7 +82,7 @@ type Downloader interface {
// Download starts new download task
Download(url string, destination string, result chan<- error)
// DownloadWithChecksum starts new download task with checksum verification
DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool)
DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int)
// Pause pauses task processing
Pause()
// Resume resumes task processing

6
cmd/mirror_update.go
View File

@@ -40,6 +40,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
}
ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
maxTries := context.Flags().Lookup("max-tries").Value.Get().(int)
verifier, err := getVerifier(context.Flags())
if err != nil {
@@ -52,7 +53,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
}
context.Progress().Printf("Downloading & parsing package files...\n")
err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch)
err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch, maxTries)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
@@ -121,7 +122,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
// In separate goroutine (to avoid blocking main), push queue to downloader
go func() {
for _, task := range queue {
context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch, maxTries)
}
// We don't need queue after this point
@@ -187,6 +188,7 @@ Example:
cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
return cmd

4
deb/remote.go
View File

@@ -403,7 +403,7 @@ ok:
// DownloadPackageIndexes downloads & parses package index files
func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
ignoreMismatch bool) error {
ignoreMismatch bool, maxTries int) error {
if repo.packageList != nil {
panic("packageList != nil")
}
@@ -433,7 +433,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
for _, info := range packagesURLs {
url, kind := info[0], info[1]
packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch)
packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch, maxTries)
if err != nil {
return err
}

73
http/download.go
View File

@@ -52,6 +52,7 @@ type downloadTask struct {
result chan<- error
expected utils.ChecksumInfo
ignoreMismatch bool
triesLeft int
}
// NewDownloader creates new instance of Downloader which specified number
@@ -127,13 +128,13 @@ func (downloader *downloaderImpl) GetProgress() aptly.Progress {
// Download starts new download task
func (downloader *downloaderImpl) Download(url string, destination string, result chan<- error) {
downloader.DownloadWithChecksum(url, destination, result, utils.ChecksumInfo{Size: -1}, false)
downloader.DownloadWithChecksum(url, destination, result, utils.ChecksumInfo{Size: -1}, false, 1)
}
// DownloadWithChecksum starts new download task with checksum verification
func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, result chan<- error,
expected utils.ChecksumInfo, ignoreMismatch bool) {
downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch}
expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) {
downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch, triesLeft: maxTries}
}
// handleTask processes single download task
@@ -153,32 +154,59 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
req.URL.RawQuery = ""
}
resp, err := downloader.client.Do(req)
var temppath string
for task.triesLeft > 0 {
temppath, err = downloader.downloadTask(req, task)
if err != nil {
task.triesLeft--
} else {
// successful download
break
}
}
// still an error after retring, giving up
if err != nil {
task.result <- err
return
}
err = os.Rename(temppath, task.destination)
if err != nil {
os.Remove(temppath)
task.result <- fmt.Errorf("%s: %s", task.url, err)
return
}
task.result <- nil
}
func (downloader *downloaderImpl) downloadTask(req *http.Request, task *downloadTask) (string, error) {
resp, err := downloader.client.Do(req)
if err != nil {
return "", fmt.Errorf("%s: %s", task.url, err)
}
if resp.Body != nil {
defer resp.Body.Close()
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
task.result <- &HTTPError{Code: resp.StatusCode, URL: task.url}
return
return "", &HTTPError{Code: resp.StatusCode, URL: task.url}
}
err = os.MkdirAll(filepath.Dir(task.destination), 0777)
if err != nil {
task.result <- fmt.Errorf("%s: %s", task.url, err)
return
return "", fmt.Errorf("%s: %s", task.url, err)
}
temppath := task.destination + ".down"
outfile, err := os.Create(temppath)
if err != nil {
task.result <- fmt.Errorf("%s: %s", task.url, err)
return
return "", fmt.Errorf("%s: %s", task.url, err)
}
defer outfile.Close()
@@ -194,8 +222,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
_, err = io.Copy(w, resp.Body)
if err != nil {
os.Remove(temppath)
task.result <- fmt.Errorf("%s: %s", task.url, err)
return
return "", fmt.Errorf("%s: %s", task.url, err)
}
if task.expected.Size != -1 {
@@ -218,20 +245,12 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
downloader.progress.Printf("WARNING: %s\n", err.Error())
} else {
os.Remove(temppath)
task.result <- err
return
return "", err
}
}
}
err = os.Rename(temppath, task.destination)
if err != nil {
os.Remove(temppath)
task.result <- fmt.Errorf("%s: %s", task.url, err)
return
}
task.result <- nil
return temppath, nil
}
// process implements download thread in goroutine
@@ -253,13 +272,13 @@ func (downloader *downloaderImpl) process() {
//
// Temporary file would be already removed, so no need to cleanup
func DownloadTemp(downloader aptly.Downloader, url string) (*os.File, error) {
return DownloadTempWithChecksum(downloader, url, utils.ChecksumInfo{Size: -1}, false)
return DownloadTempWithChecksum(downloader, url, utils.ChecksumInfo{Size: -1}, false, 1)
}
// DownloadTempWithChecksum is a DownloadTemp with checksum verification
//
// Temporary file would be already removed, so no need to cleanup
func DownloadTempWithChecksum(downloader aptly.Downloader, url string, expected utils.ChecksumInfo, ignoreMismatch bool) (*os.File, error) {
func DownloadTempWithChecksum(downloader aptly.Downloader, url string, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (*os.File, error) {
tempdir, err := ioutil.TempDir(os.TempDir(), "aptly")
if err != nil {
return nil, err
@@ -274,7 +293,7 @@ func DownloadTempWithChecksum(downloader aptly.Downloader, url string, expected
}
ch := make(chan error, 1)
downloader.DownloadWithChecksum(url, tempfile, ch, expected, ignoreMismatch)
downloader.DownloadWithChecksum(url, tempfile, ch, expected, ignoreMismatch, maxTries)
err = <-ch
@@ -311,7 +330,7 @@ var compressionMethods = []struct {
// DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until
// it finds existing file.
func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool) (io.Reader, *os.File, error) {
func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (io.Reader, *os.File, error) {
var err error
for _, method := range compressionMethods {
@@ -322,7 +341,7 @@ func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChe
for suffix, expected := range expectedChecksums {
if strings.HasSuffix(tryURL, suffix) {
file, err = DownloadTempWithChecksum(downloader, tryURL, expected, ignoreMismatch)
file, err = DownloadTempWithChecksum(downloader, tryURL, expected, ignoreMismatch, maxTries)
foundChecksum = true
break
}

4
http/fake.go
View File

@@ -59,7 +59,7 @@ func (f *FakeDownloader) Empty() bool {
}
// DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification
func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool) {
func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) {
var expectation expectedRequest
if len(f.expected) > 0 && f.expected[0].URL == url {
expectation, f.expected = f.expected[0], f.expected[1:]
@@ -116,7 +116,7 @@ func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, resul
// Download performs fake download by matching against first expectation in the queue
func (f *FakeDownloader) Download(url string, filename string, result chan<- error) {
f.DownloadWithChecksum(url, filename, result, utils.ChecksumInfo{Size: -1}, false)
f.DownloadWithChecksum(url, filename, result, utils.ChecksumInfo{Size: -1}, false, 1)
}
// Shutdown does nothing