New upstream version 1.5.0+ds1

deb/remote.go

@@ -12,7 +12,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 
@@ -33,7 +32,7 @@ const (
 
 // RemoteRepo represents remote (fetchable) Debian repository.
 //
-// Repostitory could be filtered when fetching by components, architectures
+// Repository could be filtered when fetching by components, architectures
 type RemoteRepo struct {
 	// Permanent internal ID
 	UUID string
@@ -52,7 +51,7 @@ type RemoteRepo struct {
 	// Last update date
 	LastDownloadDate time.Time
 	// Checksums for release files
-	ReleaseFiles map[string]utils.ChecksumInfo
+	ReleaseFiles map[string]utils.ChecksumInfo `json:"-"` // exclude from json output
 	// Filter for packages
 	Filter string
 	// Status marks state of repository (being updated, no action)
@@ -71,6 +70,8 @@ type RemoteRepo struct {
 	DownloadUdebs bool
 	// Should we download installer files?
 	DownloadInstaller bool
+	// Packages for json output
+	Packages []string `codec:"-" json:",omitempty"`
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 	// Parsed archived root
@@ -174,6 +175,11 @@ func (repo *RemoteRepo) RefList() *PackageRefList {
 	return repo.packageRefs
 }
 
+// PackageList returns package list for repo
+func (repo *RemoteRepo) PackageList() *PackageList {
+	return repo.packageList
+}
+
 // MarkAsUpdating puts current PID and sets status to updating
 func (repo *RemoteRepo) MarkAsUpdating() {
 	repo.Status = MirrorUpdating
@@ -423,7 +429,7 @@ ok:
 
 // DownloadPackageIndexes downloads & parses package index files
 func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, verifier pgp.Verifier, collectionFactory *CollectionFactory,
-	ignoreMismatch bool, maxTries int) error {
+	ignoreMismatch bool) error {
 	if repo.packageList != nil {
 		panic("packageList != nil")
 	}
@@ -456,7 +462,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 
 	for _, info := range packagesPaths {
 		path, kind, component, architecture := info[0], info[1], info[2], info[3]
-		packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries)
+		packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch)
 
 		isInstaller := kind == PackageTypeInstaller
 		if err != nil {
@@ -505,8 +511,10 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 		}
 		defer packagesFile.Close()
 
-		stat, _ := packagesFile.Stat()
-		progress.InitBar(stat.Size(), true)
+		if progress != nil {
+			stat, _ := packagesFile.Stat()
+			progress.InitBar(stat.Size(), true, aptly.BarMirrorUpdateBuildPackageList)
+		}
 
 		sreader := NewControlFileReader(packagesReader, false, isInstaller)
 
@@ -519,8 +527,10 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 				break
 			}
 
-			off, _ := packagesFile.Seek(0, 1)
-			progress.SetBar(int(off))
+			if progress != nil {
+				off, _ := packagesFile.Seek(0, 1)
+				progress.SetBar(int(off))
+			}
 
 			var p *Package
 
@@ -540,14 +550,20 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 				}
 			}
 			err = repo.packageList.Add(p)
-			if _, ok := err.(*PackageConflictError); ok {
-				progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
-			} else if err != nil {
-				return err
+			if err != nil {
+				if _, ok := err.(*PackageConflictError); ok {
+					if progress != nil {
+						progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
+					}
+				} else if err != nil {
+					return err
+				}
 			}
 		}
 
-		progress.ShutdownBar()
+		if progress != nil {
+			progress.ShutdownBar()
+		}
 	}
 
 	return nil
@@ -617,34 +633,44 @@ func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packag
 
 // FinalizeDownload swaps for final value of package refs
 func (repo *RemoteRepo) FinalizeDownload(collectionFactory *CollectionFactory, progress aptly.Progress) error {
+	transaction, err := collectionFactory.PackageCollection().db.OpenTransaction()
+	if err != nil {
+		return err
+	}
+	defer transaction.Discard()
+
 	repo.LastDownloadDate = time.Now()
 
 	if progress != nil {
-		progress.InitBar(int64(repo.packageList.Len()), true)
+		progress.InitBar(int64(repo.packageList.Len()), true, aptly.BarMirrorUpdateFinalizeDownload)
 	}
 
 	var i int
 
 	// update all the packages in collection
-	err := repo.packageList.ForEach(func(p *Package) error {
+	err = repo.packageList.ForEach(func(p *Package) error {
 		i++
 		if progress != nil {
 			progress.SetBar(i)
 		}
 		// download process might have updated checksums
 		p.UpdateFiles(p.Files())
-		return collectionFactory.PackageCollection().Update(p)
+		return collectionFactory.PackageCollection().UpdateInTransaction(p, transaction)
 	})
 
-	repo.packageRefs = NewPackageRefListFromPackageList(repo.packageList)
+	if err == nil {
+		repo.packageRefs = NewPackageRefListFromPackageList(repo.packageList)
+		repo.packageList = nil
+	}
 
 	if progress != nil {
 		progress.ShutdownBar()
 	}
 
-	repo.packageList = nil
-
-	return err
+	if err != nil {
+		return err
+	}
+	return transaction.Commit()
 }
 
 // Encode does msgpack encoding of RemoteRepo
@@ -696,6 +722,18 @@ func (repo *RemoteRepo) Decode(input []byte) error {
 			repo.ReleaseFiles = repo11.ReleaseFiles
 			repo.Filter = repo11.Filter
 			repo.FilterWithDeps = repo11.FilterWithDeps
+		} else if strings.Contains(err.Error(), "invalid length of bytes for decoding time") {
+			// DB created by old codec version, time.Time is not builtin type.
+			// https://github.com/ugorji/go-codec/issues/269
+			decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{
+				// only can be configured in Deprecated BasicHandle struct
+				BasicHandle: codec.BasicHandle{ // nolint: staticcheck
+					TimeNotBuiltin: true,
+				},
+			})
+			if err = decoder.Decode(repo); err != nil {
+				return err
+			}
 		} else {
 			return err
 		}
@@ -715,7 +753,6 @@ func (repo *RemoteRepo) RefKey() []byte {
 
 // RemoteRepoCollection does listing, updating/adding/deleting of RemoteRepos
 type RemoteRepoCollection struct {
-	*sync.RWMutex
 	db    database.Storage
 	cache map[string]*RemoteRepo
 }
@@ -723,9 +760,8 @@ type RemoteRepoCollection struct {
 // NewRemoteRepoCollection loads RemoteRepos from DB and makes up collection
 func NewRemoteRepoCollection(db database.Storage) *RemoteRepoCollection {
 	return &RemoteRepoCollection{
-		RWMutex: &sync.RWMutex{},
-		db:      db,
-		cache:   make(map[string]*RemoteRepo),
+		db:    db,
+		cache: make(map[string]*RemoteRepo),
 	}
 }
 
@@ -783,17 +819,13 @@ func (collection *RemoteRepoCollection) Add(repo *RemoteRepo) error {
 
 // Update stores updated information about repo in DB
 func (collection *RemoteRepoCollection) Update(repo *RemoteRepo) error {
-	err := collection.db.Put(repo.Key(), repo.Encode())
-	if err != nil {
-		return err
-	}
+	batch := collection.db.CreateBatch()
+
+	batch.Put(repo.Key(), repo.Encode())
 	if repo.packageRefs != nil {
-		err = collection.db.Put(repo.RefKey(), repo.packageRefs.Encode())
-		if err != nil {
-			return err
-		}
+		batch.Put(repo.RefKey(), repo.packageRefs.Encode())
 	}
-	return nil
+	return batch.Write()
 }
 
 // LoadComplete loads additional information for remote repo
@@ -866,16 +898,18 @@ func (collection *RemoteRepoCollection) Len() int {
 
 // Drop removes remote repo from collection
 func (collection *RemoteRepoCollection) Drop(repo *RemoteRepo) error {
-	if _, err := collection.db.Get(repo.Key()); err == database.ErrNotFound {
-		panic("repo not found!")
+	if _, err := collection.db.Get(repo.Key()); err != nil {
+		if err == database.ErrNotFound {
+			return errors.New("repo not found")
+		}
+
+		return err
 	}
 
 	delete(collection.cache, repo.UUID)
 
-	err := collection.db.Delete(repo.Key())
-	if err != nil {
-		return err
-	}
-
-	return collection.db.Delete(repo.RefKey())
+	batch := collection.db.CreateBatch()
+	batch.Delete(repo.Key())
+	batch.Delete(repo.RefKey())
+	return batch.Write()
 }