vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-07-27 13:57:46 +00:00
Code Issues Projects Releases Wiki Activity

mirror: do not download already downloaded packages

Browse Source 
this change imports downloaded packages into the pool immediately after download.
in case mirroring is aborted and later resumed, already downloaded packages will not be downloaded anymore.
This commit is contained in:
André Roth
2024-06-01 22:19:27 +02:00
parent e9bdb983c8
commit 787f954833
1 changed files with 21 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/mirror.go
+21 -30
View File
@@ -312,10 +312,9 @@ func apiMirrorsUpdate(c *gin.Context) {
b.Filter = remote.Filter b.Filter = remote.Filter
b.Architectures = remote.Architectures b.Architectures = remote.Architectures
b.Components = remote.Components b.Components = remote.Components
b.IgnoreSignatures = context.Config().GpgDisableVerify b.IgnoreSignatures = context.Config().GpgDisableVerify
log.Info().Msgf("%s: Starting mirror update\n", b.Name) log.Info().Msgf("%s: Starting mirror update", b.Name)
if c.Bind(&b) != nil { if c.Bind(&b) != nil {
return return
@@ -465,7 +464,7 @@ func apiMirrorsUpdate(c *gin.Context) {
} }
}() }()
log.Info().Msgf("%s: Spawning background processes...\n", b.Name) log.Info().Msgf("%s: Spawning background processes...", b.Name)
var wg sync.WaitGroup var wg sync.WaitGroup
for i := 0; i < context.Config().DownloadConcurrency; i++ { for i := 0; i < context.Config().DownloadConcurrency; i++ {
wg.Add(1) wg.Add(1)
@@ -501,6 +500,20 @@ func apiMirrorsUpdate(c *gin.Context) {
continue continue
} }
// and import it back to the pool
task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
if err != nil {
//return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
pushError(err)
continue
}
// update "attached" files if any
for _, additionalAtask := range task.Additional {
additionalAtask.File.PoolPath = task.File.PoolPath
additionalAtask.File.Checksums = task.File.Checksums
}
task.Done = true task.Done = true
taskFinished <- task taskFinished <- task
case <-context.Done(): case <-context.Done():
@@ -512,33 +525,11 @@ func apiMirrorsUpdate(c *gin.Context) {
} }
// Wait for all download goroutines to finish // Wait for all download goroutines to finish
log.Info().Msgf("%s: Waiting for background processes to finish...\n", b.Name) log.Info().Msgf("%s: Waiting for background processes to finish...", b.Name)
wg.Wait() wg.Wait()
log.Info().Msgf("%s: Background processes finished\n", b.Name) log.Info().Msgf("%s: Background processes finished", b.Name)
close(taskFinished) close(taskFinished)
for idx := range queue {
atask := &queue[idx]
if !atask.Done {
// download not finished yet
continue
}
// and import it back to the pool
atask.File.PoolPath, err = context.PackagePool().Import(atask.TempDownPath, atask.File.Filename, &atask.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
}
// update "attached" files if any
for _, additionalAtask := range atask.Additional {
additionalAtask.File.PoolPath = atask.File.PoolPath
additionalAtask.File.Checksums = atask.File.Checksums
}
}
select { select {
case <-context.Done(): case <-context.Done():
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: interrupted") return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: interrupted")
@@ -546,18 +537,18 @@ func apiMirrorsUpdate(c *gin.Context) {
} }
if len(errors) > 0 { if len(errors) > 0 {
log.Info().Msgf("%s: Unable to update because of previous errors\n", b.Name) log.Info().Msgf("%s: Unable to update because of previous errors", b.Name)
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: download errors:\n %s", strings.Join(errors, "\n ")) return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: download errors:\n %s", strings.Join(errors, "\n "))
} }
log.Info().Msgf("%s: Finalizing download\n", b.Name) log.Info().Msgf("%s: Finalizing download...", b.Name)
remote.FinalizeDownload(collectionFactory, out) remote.FinalizeDownload(collectionFactory, out)
err = collectionFactory.RemoteRepoCollection().Update(remote) err = collectionFactory.RemoteRepoCollection().Update(remote)
if err != nil { if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
} }
log.Info().Msgf("%s: Mirror updated successfully!\n", b.Name) log.Info().Msgf("%s: Mirror updated successfully", b.Name)
return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
}) })
} }