Add publish output progress counting remaining number of packages

api/mirror.go

@@ -59,6 +59,7 @@ func apiMirrorsCreate(c *gin.Context) {
 		Keyrings           []string
 		DownloadSources    bool
 		DownloadUdebs      bool
+		DownloadInstaller  bool
 		FilterWithDeps     bool
 		SkipComponentCheck bool
 		IgnoreSignatures   bool
@@ -92,7 +93,7 @@ func apiMirrorsCreate(c *gin.Context) {
 	}
 
 	repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures,
-		b.DownloadSources, b.DownloadUdebs)
+		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
 
 	if err != nil {
 		c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
@@ -289,7 +290,6 @@ func apiMirrorsUpdate(c *gin.Context) {
 		IgnoreSignatures     bool
 		ForceUpdate          bool
 		SkipExistingPackages bool
-		MaxTries             int
 	}
 
 	collectionFactory := context.NewCollectionFactory()
@@ -364,11 +364,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}
 
-		if b.MaxTries <= 0 {
-			b.MaxTries = 1
-		}
-
-		err = remote.DownloadPackageIndexes(out, downloader, collectionFactory, b.SkipComponentCheck, b.MaxTries)
+		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.SkipComponentCheck)
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}
@@ -388,7 +384,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
-			collectionFactory.ChecksumCollection(), b.SkipExistingPackages)
+			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}
@@ -490,8 +486,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 							remote.PackageURL(task.File.DownloadURL()).String(),
 							task.TempDownPath,
 							&task.File.Checksums,
-							b.IgnoreChecksums,
-							b.MaxTries)
+							b.IgnoreChecksums)
 						if e != nil {
 							pushError(e)
 							continue
@@ -521,7 +516,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 
 			// and import it back to the pool
-			task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection())
+			task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
 			if err != nil {
 				return fmt.Errorf("unable to import file: %s", err)
 			}

api/publish.go

@@ -183,6 +183,14 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 
 	taskName := fmt.Sprintf("Publish %s: %s", b.SourceKind, strings.Join(names, ", "))
 	task, conflictErr := runTaskInBackground(taskName, resources, func(out *task.Output, detail *task.Detail) error {
+		taskDetail := task.PublishDetail{
+			Detail: detail,
+		}
+		publishOutput := &task.PublishOutput{
+			Output:        out,
+			PublishDetail: taskDetail,
+		}
+
 		if b.Origin != "" {
 			published.Origin = b.Origin
 		}
@@ -209,7 +217,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 		}
 
-		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite)
+		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite)
 		if err != nil {
 			return fmt.Errorf("unable to publish: %s", err)
 		}

aptly/interfaces.go

@@ -97,6 +97,36 @@ type PublishedStorageProvider interface {
 	GetPublishedStorage(name string) PublishedStorage
 }
 
+// BarType used to differentiate between different progress bars
+type BarType int
+
+const (
+	// BarGeneralBuildPackageList identifies bar for building package list
+	BarGeneralBuildPackageList BarType = iota
+	// BarGeneralVerifyDependencies identifies bar for verifying dependencies
+	BarGeneralVerifyDependencies
+	// BarGeneralBuildFileList identifies bar for building file list
+	BarGeneralBuildFileList
+	// BarCleanupBuildList identifies bar for building list to cleanup
+	BarCleanupBuildList
+	// BarCleanupDeleteUnreferencedFiles identifies bar for deleting unreferenced files
+	BarCleanupDeleteUnreferencedFiles
+	// BarMirrorUpdateDownloadIndexes identifies bar for downloading index files
+	BarMirrorUpdateDownloadIndexes
+	// BarMirrorUpdateDownloadPackages identifies bar for downloading packages
+	BarMirrorUpdateDownloadPackages
+	// BarMirrorUpdateBuildPackageList identifies bar for building package list of downloaded files
+	BarMirrorUpdateBuildPackageList
+	// BarMirrorUpdateImportFiles identifies bar for importing package files
+	BarMirrorUpdateImportFiles
+	// BarMirrorUpdateFinalizeDownload identifies bar for finalizing downloads
+	BarMirrorUpdateFinalizeDownload
+	// BarPublishGeneratePackageFiles identifies bar for generating package files to publish
+	BarPublishGeneratePackageFiles
+	// BarPublishFinalizeIndexes identifies bar for finalizing index files
+	BarPublishFinalizeIndexes
+)
+
 // Progress is a progress displaying entity, it allows progress bars & simple prints
 type Progress interface {
 	// Writer interface to support progress bar ticking
@@ -108,7 +138,7 @@ type Progress interface {
 	// Flush returns when all queued messages are sent
 	Flush()
 	// InitBar starts progressbar for count bytes or count items
-	InitBar(count int64, isBytes bool)
+	InitBar(count int64, isBytes bool, barType BarType)
 	// ShutdownBar stops progress bar and hides it
 	ShutdownBar()
 	// AddBar increments progress for progress bar

cmd/db_cleanup.go

@@ -5,6 +5,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
@@ -203,7 +204,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	// now, build a list of files that should be present in Repository (package pool)
 	context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())
-	context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
+	context.Progress().InitBar(int64(existingPackageRefs.Len()), false, aptly.BarCleanupBuildList)
 
 	err = existingPackageRefs.ForEach(func(key []byte) error {
 		pkg, err2 := collectionFactory.PackageCollection().ByKey(key)
@@ -257,7 +258,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		}
 
 		if !dryRun {
-			context.Progress().InitBar(int64(len(filesToDelete)), false)
+			context.Progress().InitBar(int64(len(filesToDelete)), false, aptly.BarCleanupDeleteUnreferencedFiles)
 
 			var size, totalSize int64
 			for _, file := range filesToDelete {

cmd/mirror_update.go

@@ -117,7 +117,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
 
 	// Download from the queue
-	context.Progress().InitBar(downloadSize, true)
+	context.Progress().InitBar(downloadSize, true, aptly.BarMirrorUpdateDownloadPackages)
 
 	downloadQueue := make(chan int)
 
@@ -198,7 +198,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	// Import downloaded files
-	context.Progress().InitBar(int64(len(queue)), false)
+	context.Progress().InitBar(int64(len(queue)), false, aptly.BarMirrorUpdateImportFiles)
 
 	for idx := range queue {
 		context.Progress().AddBar(1)

console/progress.go

@@ -69,7 +69,7 @@ func (p *Progress) Flush() {
 }
 
 // InitBar starts progressbar for count bytes or count items
-func (p *Progress) InitBar(count int64, isBytes bool) {
+func (p *Progress) InitBar(count int64, isBytes bool, barType aptly.BarType) {
 	if p.bar != nil {
 		panic("bar already initialized")
 	}

deb/index_files.go

@@ -391,7 +391,7 @@ func (files *indexFiles) ReleaseFile() *indexFile {
 
 func (files *indexFiles) FinalizeAll(progress aptly.Progress, signer pgp.Signer) (err error) {
 	if progress != nil {
-		progress.InitBar(int64(len(files.indexes)), false)
+		progress.InitBar(int64(len(files.indexes)), false, aptly.BarPublishFinalizeIndexes)
 		defer progress.ShutdownBar()
 	}
 

deb/list.go

@@ -99,7 +99,7 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 	result := NewPackageListWithDuplicates(false, reflist.Len())
 
 	if progress != nil {
-		progress.InitBar(int64(reflist.Len()), false)
+		progress.InitBar(int64(reflist.Len()), false, aptly.BarGeneralBuildPackageList)
 	}
 
 	err := reflist.ForEach(func(key []byte) error {
@@ -314,7 +314,7 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 	missing := make([]Dependency, 0, 128)
 
 	if progress != nil {
-		progress.InitBar(int64(l.Len())*int64(len(architectures)), false)
+		progress.InitBar(int64(l.Len())*int64(len(architectures)), false, aptly.BarGeneralVerifyDependencies)
 	}
 
 	for _, arch := range architectures {

deb/publish.go

@@ -594,7 +594,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	}
 
 	if progress != nil {
-		progress.InitBar(count, false)
+		progress.InitBar(count, false, aptly.BarPublishGeneratePackageFiles)
 	}
 
 	for component, list := range lists {

deb/remote.go

@@ -513,7 +513,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 
 		if progress != nil {
 			stat, _ := packagesFile.Stat()
-			progress.InitBar(stat.Size(), true)
+			progress.InitBar(stat.Size(), true, aptly.BarMirrorUpdateBuildPackageList)
 		}
 
 		sreader := NewControlFileReader(packagesReader, false, isInstaller)
@@ -642,7 +642,7 @@ func (repo *RemoteRepo) FinalizeDownload(collectionFactory *CollectionFactory, p
 	repo.LastDownloadDate = time.Now()
 
 	if progress != nil {
-		progress.InitBar(int64(repo.packageList.Len()), false)
+		progress.InitBar(int64(repo.packageList.Len()), true, aptly.BarMirrorUpdateFinalizeDownload)
 	}
 
 	var i int

files/package_pool.go

@@ -94,7 +94,7 @@ func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error)
 	}
 
 	if progress != nil {
-		progress.InitBar(int64(len(dirs)), false)
+		progress.InitBar(int64(len(dirs)), false, aptly.BarGeneralBuildFileList)
 		defer progress.ShutdownBar()
 	}
 

http/temp.go

@@ -30,7 +30,7 @@ func DownloadTempWithChecksum(ctx context.Context, downloader aptly.Downloader,
 	tempfile := filepath.Join(tempdir, "buffer")
 
 	if expected != nil && downloader.GetProgress() != nil {
-		downloader.GetProgress().InitBar(expected.Size, true)
+		downloader.GetProgress().InitBar(expected.Size, true, aptly.BarMirrorUpdateDownloadIndexes)
 		defer downloader.GetProgress().ShutdownBar()
 	}
 

system/t12_api/publish.py

@@ -153,6 +153,12 @@ class PublishSnapshotAPITest(APITest):
             }
         )
         self.check_equal(resp.json()['State'], 2)
+
+        _id = resp.json()['ID']
+        resp = self.get("/api/tasks/" + str(_id) + "/detail")
+        self.check_equal(resp.json()['RemainingNumberOfPackages'], 0)
+        self.check_equal(resp.json()['TotalNumberOfPackages'], 1)
+
         repo_expected = {
             'AcquireByHash': True,
             'Architectures': ['i386'],

task/output.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"fmt"
 	"sync"
+
+	"github.com/aptly-dev/aptly/aptly"
 )
 
 // Output represents a safe standard output of task
@@ -13,6 +15,13 @@ type Output struct {
 	output *bytes.Buffer
 }
 
+// PublishOutput specific output for publishing api
+type PublishOutput struct {
+	*Output
+	PublishDetail
+	barType *aptly.BarType
+}
+
 // NewOutput creates new output
 func NewOutput() *Output {
 	return &Output{mu: &sync.Mutex{}, output: &bytes.Buffer{}}
@@ -54,20 +63,43 @@ func (t *Output) Flush() {
 }
 
 // InitBar is needed for progress compatibility
-func (t *Output) InitBar(count int64, isBytes bool) {
+func (t *Output) InitBar(count int64, isBytes bool, barType aptly.BarType) {
 	// Not implemented
 }
 
+// InitBar publish output specific
+func (t *PublishOutput) InitBar(count int64, isBytes bool, barType aptly.BarType) {
+	t.barType = &barType
+	if barType == aptly.BarPublishGeneratePackageFiles {
+		t.TotalNumberOfPackages = count
+		t.RemainingNumberOfPackages = count
+		t.Store(t)
+	}
+}
+
 // ShutdownBar is needed for progress compatibility
 func (t *Output) ShutdownBar() {
 	// Not implemented
 }
 
+// ShutdownBar publish output specific
+func (t *PublishOutput) ShutdownBar() {
+	t.barType = nil
+}
+
 // AddBar is needed for progress compatibility
 func (t *Output) AddBar(count int) {
 	// Not implemented
 }
 
+// AddBar publish output specific
+func (t *PublishOutput) AddBar(count int) {
+	if t.barType != nil && *t.barType == aptly.BarPublishGeneratePackageFiles {
+		t.RemainingNumberOfPackages--
+		t.Store(t)
+	}
+}
+
 // SetBar sets current position for progress bar
 func (t *Output) SetBar(count int) {
 	// Not implemented

task/task.go

@@ -12,6 +12,13 @@ type Detail struct {
 	atomic.Value
 }
 
+// PublishDetail represents publish task details
+type PublishDetail struct {
+	*Detail
+	TotalNumberOfPackages     int64
+	RemainingNumberOfPackages int64
+}
+
 // Process is a function implementing the actual task logic
 type Process func(out *Output, detail *Detail) error