From f09a273ad761537a1876ec3e74487407918a99cb Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Thu, 17 May 2018 15:44:49 +0200 Subject: [PATCH] Add publish output progress counting remaining number of packages --- api/mirror.go | 17 ++++++----------- api/publish.go | 10 +++++++++- aptly/interfaces.go | 32 +++++++++++++++++++++++++++++++- cmd/db_cleanup.go | 5 +++-- cmd/mirror_update.go | 4 ++-- console/progress.go | 2 +- deb/index_files.go | 2 +- deb/list.go | 4 ++-- deb/publish.go | 2 +- deb/remote.go | 4 ++-- files/package_pool.go | 2 +- http/temp.go | 2 +- system/t12_api/publish.py | 6 ++++++ task/output.go | 34 +++++++++++++++++++++++++++++++++- task/task.go | 7 +++++++ 15 files changed, 106 insertions(+), 27 deletions(-) diff --git a/api/mirror.go b/api/mirror.go index b33dc53f..5454b76f 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -59,6 +59,7 @@ func apiMirrorsCreate(c *gin.Context) { Keyrings []string DownloadSources bool DownloadUdebs bool + DownloadInstaller bool FilterWithDeps bool SkipComponentCheck bool IgnoreSignatures bool @@ -92,7 +93,7 @@ func apiMirrorsCreate(c *gin.Context) { } repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures, - b.DownloadSources, b.DownloadUdebs) + b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller) if err != nil { c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err)) @@ -289,7 +290,6 @@ func apiMirrorsUpdate(c *gin.Context) { IgnoreSignatures bool ForceUpdate bool SkipExistingPackages bool - MaxTries int } collectionFactory := context.NewCollectionFactory() @@ -364,11 +364,7 @@ func apiMirrorsUpdate(c *gin.Context) { } } - if b.MaxTries <= 0 { - b.MaxTries = 1 - } - - err = remote.DownloadPackageIndexes(out, downloader, collectionFactory, b.SkipComponentCheck, b.MaxTries) + err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.SkipComponentCheck) if err != nil { return fmt.Errorf("unable to update: %s", err) } @@ -388,7 +384,7 @@ func apiMirrorsUpdate(c *gin.Context) { } queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(), - collectionFactory.ChecksumCollection(), b.SkipExistingPackages) + collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages) if err != nil { return fmt.Errorf("unable to update: %s", err) } @@ -490,8 +486,7 @@ func apiMirrorsUpdate(c *gin.Context) { remote.PackageURL(task.File.DownloadURL()).String(), task.TempDownPath, &task.File.Checksums, - b.IgnoreChecksums, - b.MaxTries) + b.IgnoreChecksums) if e != nil { pushError(e) continue @@ -521,7 +516,7 @@ func apiMirrorsUpdate(c *gin.Context) { } // and import it back to the pool - task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection()) + task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil)) if err != nil { return fmt.Errorf("unable to import file: %s", err) } diff --git a/api/publish.go b/api/publish.go index 557f40db..c04b22d1 100644 --- a/api/publish.go +++ b/api/publish.go @@ -183,6 +183,14 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { taskName := fmt.Sprintf("Publish %s: %s", b.SourceKind, strings.Join(names, ", ")) task, conflictErr := runTaskInBackground(taskName, resources, func(out *task.Output, detail *task.Detail) error { + taskDetail := task.PublishDetail{ + Detail: detail, + } + publishOutput := &task.PublishOutput{ + Output: out, + PublishDetail: taskDetail, + } + if b.Origin != "" { published.Origin = b.Origin } @@ -209,7 +217,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate) } - err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite) + err := published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/aptly/interfaces.go b/aptly/interfaces.go index 1b529619..e912daba 100644 --- a/aptly/interfaces.go +++ b/aptly/interfaces.go @@ -97,6 +97,36 @@ type PublishedStorageProvider interface { GetPublishedStorage(name string) PublishedStorage } +// BarType used to differentiate between different progress bars +type BarType int + +const ( + // BarGeneralBuildPackageList identifies bar for building package list + BarGeneralBuildPackageList BarType = iota + // BarGeneralVerifyDependencies identifies bar for verifying dependencies + BarGeneralVerifyDependencies + // BarGeneralBuildFileList identifies bar for building file list + BarGeneralBuildFileList + // BarCleanupBuildList identifies bar for building list to cleanup + BarCleanupBuildList + // BarCleanupDeleteUnreferencedFiles identifies bar for deleting unreferenced files + BarCleanupDeleteUnreferencedFiles + // BarMirrorUpdateDownloadIndexes identifies bar for downloading index files + BarMirrorUpdateDownloadIndexes + // BarMirrorUpdateDownloadPackages identifies bar for downloading packages + BarMirrorUpdateDownloadPackages + // BarMirrorUpdateBuildPackageList identifies bar for building package list of downloaded files + BarMirrorUpdateBuildPackageList + // BarMirrorUpdateImportFiles identifies bar for importing package files + BarMirrorUpdateImportFiles + // BarMirrorUpdateFinalizeDownload identifies bar for finalizing downloads + BarMirrorUpdateFinalizeDownload + // BarPublishGeneratePackageFiles identifies bar for generating package files to publish + BarPublishGeneratePackageFiles + // BarPublishFinalizeIndexes identifies bar for finalizing index files + BarPublishFinalizeIndexes +) + // Progress is a progress displaying entity, it allows progress bars & simple prints type Progress interface { // Writer interface to support progress bar ticking @@ -108,7 +138,7 @@ type Progress interface { // Flush returns when all queued messages are sent Flush() // InitBar starts progressbar for count bytes or count items - InitBar(count int64, isBytes bool) + InitBar(count int64, isBytes bool, barType BarType) // ShutdownBar stops progress bar and hides it ShutdownBar() // AddBar increments progress for progress bar diff --git a/cmd/db_cleanup.go b/cmd/db_cleanup.go index 5586dd75..a8905169 100644 --- a/cmd/db_cleanup.go +++ b/cmd/db_cleanup.go @@ -5,6 +5,7 @@ import ( "sort" "strings" + "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/deb" "github.com/aptly-dev/aptly/utils" "github.com/smira/commander" @@ -203,7 +204,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error { // now, build a list of files that should be present in Repository (package pool) context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|") referencedFiles := make([]string, 0, existingPackageRefs.Len()) - context.Progress().InitBar(int64(existingPackageRefs.Len()), false) + context.Progress().InitBar(int64(existingPackageRefs.Len()), false, aptly.BarCleanupBuildList) err = existingPackageRefs.ForEach(func(key []byte) error { pkg, err2 := collectionFactory.PackageCollection().ByKey(key) @@ -257,7 +258,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error { } if !dryRun { - context.Progress().InitBar(int64(len(filesToDelete)), false) + context.Progress().InitBar(int64(len(filesToDelete)), false, aptly.BarCleanupDeleteUnreferencedFiles) var size, totalSize int64 for _, file := range filesToDelete { diff --git a/cmd/mirror_update.go b/cmd/mirror_update.go index dfbf0718..614e61d3 100644 --- a/cmd/mirror_update.go +++ b/cmd/mirror_update.go @@ -117,7 +117,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize)) // Download from the queue - context.Progress().InitBar(downloadSize, true) + context.Progress().InitBar(downloadSize, true, aptly.BarMirrorUpdateDownloadPackages) downloadQueue := make(chan int) @@ -198,7 +198,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { } // Import downloaded files - context.Progress().InitBar(int64(len(queue)), false) + context.Progress().InitBar(int64(len(queue)), false, aptly.BarMirrorUpdateImportFiles) for idx := range queue { context.Progress().AddBar(1) diff --git a/console/progress.go b/console/progress.go index 56abbef8..3671fc3a 100644 --- a/console/progress.go +++ b/console/progress.go @@ -69,7 +69,7 @@ func (p *Progress) Flush() { } // InitBar starts progressbar for count bytes or count items -func (p *Progress) InitBar(count int64, isBytes bool) { +func (p *Progress) InitBar(count int64, isBytes bool, barType aptly.BarType) { if p.bar != nil { panic("bar already initialized") } diff --git a/deb/index_files.go b/deb/index_files.go index 4affd0a9..5e711d90 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -391,7 +391,7 @@ func (files *indexFiles) ReleaseFile() *indexFile { func (files *indexFiles) FinalizeAll(progress aptly.Progress, signer pgp.Signer) (err error) { if progress != nil { - progress.InitBar(int64(len(files.indexes)), false) + progress.InitBar(int64(len(files.indexes)), false, aptly.BarPublishFinalizeIndexes) defer progress.ShutdownBar() } diff --git a/deb/list.go b/deb/list.go index 263622b7..ec14bb98 100644 --- a/deb/list.go +++ b/deb/list.go @@ -99,7 +99,7 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle result := NewPackageListWithDuplicates(false, reflist.Len()) if progress != nil { - progress.InitBar(int64(reflist.Len()), false) + progress.InitBar(int64(reflist.Len()), false, aptly.BarGeneralBuildPackageList) } err := reflist.ForEach(func(key []byte) error { @@ -314,7 +314,7 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so missing := make([]Dependency, 0, 128) if progress != nil { - progress.InitBar(int64(l.Len())*int64(len(architectures)), false) + progress.InitBar(int64(l.Len())*int64(len(architectures)), false, aptly.BarGeneralVerifyDependencies) } for _, arch := range architectures { diff --git a/deb/publish.go b/deb/publish.go index 30673ac9..e6332050 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -594,7 +594,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } if progress != nil { - progress.InitBar(count, false) + progress.InitBar(count, false, aptly.BarPublishGeneratePackageFiles) } for component, list := range lists { diff --git a/deb/remote.go b/deb/remote.go index 0b7d71e5..8164b0a2 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -513,7 +513,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly. if progress != nil { stat, _ := packagesFile.Stat() - progress.InitBar(stat.Size(), true) + progress.InitBar(stat.Size(), true, aptly.BarMirrorUpdateBuildPackageList) } sreader := NewControlFileReader(packagesReader, false, isInstaller) @@ -642,7 +642,7 @@ func (repo *RemoteRepo) FinalizeDownload(collectionFactory *CollectionFactory, p repo.LastDownloadDate = time.Now() if progress != nil { - progress.InitBar(int64(repo.packageList.Len()), false) + progress.InitBar(int64(repo.packageList.Len()), true, aptly.BarMirrorUpdateFinalizeDownload) } var i int diff --git a/files/package_pool.go b/files/package_pool.go index 8aa3a084..8b1e8b25 100644 --- a/files/package_pool.go +++ b/files/package_pool.go @@ -94,7 +94,7 @@ func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) } if progress != nil { - progress.InitBar(int64(len(dirs)), false) + progress.InitBar(int64(len(dirs)), false, aptly.BarGeneralBuildFileList) defer progress.ShutdownBar() } diff --git a/http/temp.go b/http/temp.go index dc98ab8e..8e6a3fde 100644 --- a/http/temp.go +++ b/http/temp.go @@ -30,7 +30,7 @@ func DownloadTempWithChecksum(ctx context.Context, downloader aptly.Downloader, tempfile := filepath.Join(tempdir, "buffer") if expected != nil && downloader.GetProgress() != nil { - downloader.GetProgress().InitBar(expected.Size, true) + downloader.GetProgress().InitBar(expected.Size, true, aptly.BarMirrorUpdateDownloadIndexes) defer downloader.GetProgress().ShutdownBar() } diff --git a/system/t12_api/publish.py b/system/t12_api/publish.py index 98874561..9d8059d6 100644 --- a/system/t12_api/publish.py +++ b/system/t12_api/publish.py @@ -153,6 +153,12 @@ class PublishSnapshotAPITest(APITest): } ) self.check_equal(resp.json()['State'], 2) + + _id = resp.json()['ID'] + resp = self.get("/api/tasks/" + str(_id) + "/detail") + self.check_equal(resp.json()['RemainingNumberOfPackages'], 0) + self.check_equal(resp.json()['TotalNumberOfPackages'], 1) + repo_expected = { 'AcquireByHash': True, 'Architectures': ['i386'], diff --git a/task/output.go b/task/output.go index 4f451df3..57d4d85c 100644 --- a/task/output.go +++ b/task/output.go @@ -4,6 +4,8 @@ import ( "bytes" "fmt" "sync" + + "github.com/aptly-dev/aptly/aptly" ) // Output represents a safe standard output of task @@ -13,6 +15,13 @@ type Output struct { output *bytes.Buffer } +// PublishOutput specific output for publishing api +type PublishOutput struct { + *Output + PublishDetail + barType *aptly.BarType +} + // NewOutput creates new output func NewOutput() *Output { return &Output{mu: &sync.Mutex{}, output: &bytes.Buffer{}} @@ -54,20 +63,43 @@ func (t *Output) Flush() { } // InitBar is needed for progress compatibility -func (t *Output) InitBar(count int64, isBytes bool) { +func (t *Output) InitBar(count int64, isBytes bool, barType aptly.BarType) { // Not implemented } +// InitBar publish output specific +func (t *PublishOutput) InitBar(count int64, isBytes bool, barType aptly.BarType) { + t.barType = &barType + if barType == aptly.BarPublishGeneratePackageFiles { + t.TotalNumberOfPackages = count + t.RemainingNumberOfPackages = count + t.Store(t) + } +} + // ShutdownBar is needed for progress compatibility func (t *Output) ShutdownBar() { // Not implemented } +// ShutdownBar publish output specific +func (t *PublishOutput) ShutdownBar() { + t.barType = nil +} + // AddBar is needed for progress compatibility func (t *Output) AddBar(count int) { // Not implemented } +// AddBar publish output specific +func (t *PublishOutput) AddBar(count int) { + if t.barType != nil && *t.barType == aptly.BarPublishGeneratePackageFiles { + t.RemainingNumberOfPackages-- + t.Store(t) + } +} + // SetBar sets current position for progress bar func (t *Output) SetBar(count int) { // Not implemented diff --git a/task/task.go b/task/task.go index 362d1351..52cb51bc 100644 --- a/task/task.go +++ b/task/task.go @@ -12,6 +12,13 @@ type Detail struct { atomic.Value } +// PublishDetail represents publish task details +type PublishDetail struct { + *Detail + TotalNumberOfPackages int64 + RemainingNumberOfPackages int64 +} + // Process is a function implementing the actual task logic type Process func(out *Output, detail *Detail) error