Merge pull request #683 from smira/545-download-contxt

Use Go context to abort gracefully mirror updates

.travis.yml

@@ -4,9 +4,9 @@ sudo: required
 language: go
 
 go:
-  - 1.6.x
   - 1.7.x
   - 1.8.x
+  - 1.9.x
   - master
 
 go_import_path: github.com/smira/aptly

AUTHORS

@@ -28,3 +28,5 @@ List of contributors, in chronological order:
 * Charles Hsu (https://github.com/charz)
 * Clemens Rabe (https://github.com/seeraven)
 * TJ Merritt (https://github.com/tjmerritt)
+* Matt Martyn (https://github.com/MMartyn)
+* Ludovico Cavedon (https://github.com/cavedon)

Gopkg.lock

@@ -1,4 +1,4 @@
-memo = "becdf010a814559719c990c1bd645c737cee332ad52004c440605c13de100d45"
+memo = "57879f27cc9f82276b92ed638fbc04122c3793ed4a16bea668c9fbfda280c280"
 
 [[projects]]
   name = "github.com/AlekSi/pointer"
@@ -20,9 +20,9 @@ memo = "becdf010a814559719c990c1bd645c737cee332ad52004c440605c13de100d45"
 
 [[projects]]
   name = "github.com/aws/aws-sdk-go"
-  packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/s3","service/sts"]
-  revision = "2db5849d2939d93075d911138309a83235032bea"
-  version = "v1.8.0"
+  packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","internal/shareddefaults","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/s3","service/sts"]
+  revision = "c652f9369083515c3ddf1fbaf6df68da2c101545"
+  version = "v1.12.1"
 
 [[projects]]
   name = "github.com/cheggaaa/pb"
@@ -30,10 +30,17 @@ memo = "becdf010a814559719c990c1bd645c737cee332ad52004c440605c13de100d45"
   revision = "cdf719fac0dd208251aa828e687c2d5802053b51"
   version = "v1.0.10"
 
+[[projects]]
+  branch = "master"
+  name = "github.com/gin-contrib/sse"
+  packages = ["."]
+  revision = "22d885f9ecc78bf4ee5d72b937e4bbcdc58e8cae"
+
 [[projects]]
   name = "github.com/gin-gonic/gin"
   packages = [".","binding","render"]
-  revision = "b1758d3bfa09e61ddbc1c9a627e936eec6a170de"
+  revision = "d459835d2b077e44f7c9b453505ee29881d5d12d"
+  version = "v1.2"
 
 [[projects]]
   name = "github.com/go-ini/ini"
@@ -41,6 +48,12 @@ memo = "becdf010a814559719c990c1bd645c737cee332ad52004c440605c13de100d45"
   revision = "1730955e3146956d6a087861380f9b4667ed5071"
   version = "v1.26.0"
 
+[[projects]]
+  branch = "master"
+  name = "github.com/golang/protobuf"
+  packages = ["proto"]
+  revision = "130e6b02ab059e7b717a096f397c5b60111cae74"
+
 [[projects]]
   branch = "master"
   name = "github.com/golang/snappy"
@@ -66,10 +79,10 @@ memo = "becdf010a814559719c990c1bd645c737cee332ad52004c440605c13de100d45"
   version = "0.2.2"
 
 [[projects]]
-  name = "github.com/julienschmidt/httprouter"
+  name = "github.com/mattn/go-isatty"
   packages = ["."]
-  revision = "8c199fb6259ffc1af525cc3ad52ee60ba8359669"
-  version = "v1.1"
+  revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
+  version = "v0.0.3"
 
 [[projects]]
   name = "github.com/mattn/go-runewidth"
@@ -117,7 +130,7 @@ memo = "becdf010a814559719c990c1bd645c737cee332ad52004c440605c13de100d45"
   branch = "master"
   name = "github.com/smira/flag"
   packages = ["."]
-  revision = "357ed3e599ffcbd4aeaa828e1d10da2df3ea5107"
+  revision = "695ea5e84e76dea7c8656e43c384e54b32aa1b2a"
 
 [[projects]]
   branch = "master"
@@ -153,7 +166,7 @@ memo = "becdf010a814559719c990c1bd645c737cee332ad52004c440605c13de100d45"
   branch = "master"
   name = "github.com/syndtr/goleveldb"
   packages = ["leveldb","leveldb/cache","leveldb/comparer","leveldb/errors","leveldb/filter","leveldb/iterator","leveldb/journal","leveldb/memdb","leveldb/opt","leveldb/storage","leveldb/table","leveldb/util"]
-  revision = "3c5717caf1475fd25964109a0fc640bd150fce43"
+  revision = "549b6d6b1c0419617182954dd77770f2e2685ed5"
 
 [[projects]]
   name = "github.com/ugorji/go"
@@ -184,8 +197,20 @@ memo = "becdf010a814559719c990c1bd645c737cee332ad52004c440605c13de100d45"
   packages = ["."]
   revision = "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec"
 
+[[projects]]
+  name = "gopkg.in/go-playground/validator.v8"
+  packages = ["."]
+  revision = "5f1438d3fca68893a817e4a66806cea46a9e4ebf"
+  version = "v8.18.2"
+
 [[projects]]
   name = "gopkg.in/h2non/filetype.v1"
   packages = ["types"]
   revision = "3093b8ebec6efb56ac813238b8beab4ed4eaac6a"
   version = "v1.0.1"
+
+[[projects]]
+  branch = "v2"
+  name = "gopkg.in/yaml.v2"
+  packages = ["."]
+  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"

Gopkg.toml

@@ -1,8 +1,4 @@
 
-[[dependencies]]
-  name = "github.com/gin-gonic/gin"
-  revision = "b1758d3bfa09e61ddbc1c9a627e936eec6a170de"
-
 [[dependencies]]
   branch = "master"
   name = "github.com/mkrautz/goar"

README.rst

@@ -64,7 +64,7 @@ If you would like to use nightly builds (unstable), please use following reposit
 
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.6+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.7+ required)::
 
     mkdir -p $GOPATH/src/github.com/smira/aptly
     git clone https://github.com/smira/aptly $GOPATH/src/github.com/smira/aptly

api/api.go

@@ -112,7 +112,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 
 	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), nil)
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
@@ -120,7 +120,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 	if queryS != "" {
 		q, err := query.Parse(c.Request.URL.Query().Get("q"))
 		if err != nil {
-			c.Fail(400, err)
+			c.AbortWithError(400, err)
 			return
 		}
 
@@ -137,7 +137,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 			sort.Strings(architecturesList)
 
 			if len(architecturesList) == 0 {
-				c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
 				return
 			}
 		}
@@ -147,7 +147,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
 			nil, context.DependencyOptions(), architecturesList)
 		if err != nil {
-			c.Fail(500, fmt.Errorf("unable to search: %s", err))
+			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
 			return
 		}
 	}

api/files.go

@@ -24,7 +24,7 @@ func verifyPath(path string) bool {
 
 func verifyDir(c *gin.Context) bool {
 	if !verifyPath(c.Params.ByName("dir")) {
-		c.Fail(400, fmt.Errorf("wrong dir"))
+		c.AbortWithError(400, fmt.Errorf("wrong dir"))
 		return false
 	}
 
@@ -53,7 +53,7 @@ func apiFilesListDirs(c *gin.Context) {
 	})
 
 	if err != nil && !os.IsNotExist(err) {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -70,13 +70,13 @@ func apiFilesUpload(c *gin.Context) {
 	err := os.MkdirAll(path, 0777)
 
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -86,7 +86,7 @@ func apiFilesUpload(c *gin.Context) {
 		for _, file := range files {
 			src, err := file.Open()
 			if err != nil {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 				return
 			}
 			defer src.Close()
@@ -94,14 +94,14 @@ func apiFilesUpload(c *gin.Context) {
 			destPath := filepath.Join(path, filepath.Base(file.Filename))
 			dst, err := os.Create(destPath)
 			if err != nil {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 				return
 			}
 			defer dst.Close()
 
 			_, err = io.Copy(dst, src)
 			if err != nil {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 				return
 			}
 
@@ -138,9 +138,9 @@ func apiFilesListFiles(c *gin.Context) {
 
 	if err != nil {
 		if os.IsNotExist(err) {
-			c.Fail(404, err)
+			c.AbortWithError(404, err)
 		} else {
-			c.Fail(500, err)
+			c.AbortWithError(500, err)
 		}
 		return
 	}
@@ -156,7 +156,7 @@ func apiFilesDeleteDir(c *gin.Context) {
 
 	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -170,14 +170,14 @@ func apiFilesDeleteFile(c *gin.Context) {
 	}
 
 	if !verifyPath(c.Params.ByName("name")) {
-		c.Fail(400, fmt.Errorf("wrong file"))
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
 		return
 	}
 
 	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
 	if err != nil {
 		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
-			c.Fail(500, err)
+			c.AbortWithError(500, err)
 			return
 		}
 	}

api/graph.go

@@ -53,25 +53,25 @@ func apiGraph(c *gin.Context) {
 
 	stdin, err := command.StdinPipe()
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	_, err = io.Copy(stdin, buf)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	err = stdin.Close()
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	output, err = command.Output()
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		c.AbortWithError(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
 		return
 	}
 

api/packages.go

@@ -8,7 +8,7 @@ import (
 func apiPackagesShow(c *gin.Context) {
 	p, err := context.CollectionFactory().PackageCollection().ByKey([]byte(c.Params.ByName("key")))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 

api/publish.go

@@ -77,7 +77,7 @@ func apiPublishList(c *gin.Context) {
 	})
 
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -106,18 +106,18 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		Signing              SigningOptions
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
 	signer, err := getSigner(&b.Signing)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
 		return
 	}
 
 	if len(b.Sources) == 0 {
-		c.Fail(400, fmt.Errorf("unable to publish: soures are empty"))
+		c.AbortWithError(400, fmt.Errorf("unable to publish: soures are empty"))
 		return
 	}
 
@@ -136,13 +136,13 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 
 			snapshot, err = snapshotCollection.ByName(source.Name)
 			if err != nil {
-				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
 				return
 			}
 
 			err = snapshotCollection.LoadComplete(snapshot)
 			if err != nil {
-				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 				return
 			}
 
@@ -160,19 +160,19 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 
 			localRepo, err = localCollection.ByName(source.Name)
 			if err != nil {
-				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
 				return
 			}
 
 			err = localCollection.LoadComplete(localRepo)
 			if err != nil {
-				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 			}
 
 			sources = append(sources, localRepo)
 		}
 	} else {
-		c.Fail(400, fmt.Errorf("unknown SourceKind"))
+		c.AbortWithError(400, fmt.Errorf("unknown SourceKind"))
 		return
 	}
 
@@ -182,7 +182,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 
 	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, context.CollectionFactory())
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 		return
 	}
 	if b.Origin != "" {
@@ -204,19 +204,19 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 	duplicate := collection.CheckDuplicate(published)
 	if duplicate != nil {
 		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
-		c.Fail(400, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate))
+		c.AbortWithError(400, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate))
 		return
 	}
 
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 		return
 	}
 
 	err = collection.Add(published)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to save to DB: %s", err))
 		return
 	}
 
@@ -233,19 +233,21 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		ForceOverwrite bool
 		Signing        SigningOptions
 		SkipContents   *bool
+		SkipCleanup    *bool
 		Snapshots      []struct {
 			Component string `binding:"required"`
 			Name      string `binding:"required"`
 		}
+		AcquireByHash *bool
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
 	signer, err := getSigner(&b.Signing)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
 		return
 	}
 
@@ -264,12 +266,12 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
-		c.Fail(404, fmt.Errorf("unable to update: %s", err))
+		c.AbortWithError(404, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 	err = collection.LoadComplete(published, context.CollectionFactory())
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
@@ -277,7 +279,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 
 	if published.SourceKind == deb.SourceLocalRepo {
 		if len(b.Snapshots) > 0 {
-			c.Fail(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
+			c.AbortWithError(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
 			return
 		}
 		updatedComponents = published.Components()
@@ -288,19 +290,19 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		publishedComponents := published.Components()
 		for _, snapshotInfo := range b.Snapshots {
 			if !utils.StrSliceHasItem(publishedComponents, snapshotInfo.Component) {
-				c.Fail(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
+				c.AbortWithError(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
 				return
 			}
 
 			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
 			if err != nil {
-				c.Fail(404, err2)
+				c.AbortWithError(404, err2)
 				return
 			}
 
 			err2 = snapshotCollection.LoadComplete(snapshot)
 			if err2 != nil {
-				c.Fail(500, err2)
+				c.AbortWithError(500, err2)
 				return
 			}
 
@@ -308,7 +310,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			updatedComponents = append(updatedComponents, snapshotInfo.Component)
 		}
 	} else {
-		c.Fail(500, fmt.Errorf("unknown published repository type"))
+		c.AbortWithError(500, fmt.Errorf("unknown published repository type"))
 		return
 	}
 
@@ -316,23 +318,29 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		published.SkipContents = *b.SkipContents
 	}
 
+	if b.AcquireByHash != nil {
+		published.AcquireByHash = *b.AcquireByHash
+	}
+
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
 	err = collection.Update(published)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to save to DB: %s", err))
 		return
 	}
 
-	err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
-		context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
-	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to update: %s", err))
-		return
+	if b.SkipCleanup == nil || !*b.SkipCleanup {
+		err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
+			context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
+		if err != nil {
+			c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
+			return
+		}
 	}
 
 	c.JSON(200, published)
@@ -341,6 +349,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 // DELETE /publish/:prefix/:distribution
 func apiPublishDrop(c *gin.Context) {
 	force := c.Request.URL.Query().Get("force") == "1"
+	skipCleanup := c.Request.URL.Query().Get("SkipCleanup") == "1"
 
 	param := parseEscapedPath(c.Params.ByName("prefix"))
 	storage, prefix := deb.ParsePrefix(param)
@@ -356,9 +365,9 @@ func apiPublishDrop(c *gin.Context) {
 	defer collection.Unlock()
 
 	err := collection.Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress(), force)
+		context.CollectionFactory(), context.Progress(), force, skipCleanup)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to drop: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to drop: %s", err))
 		return
 	}
 

api/repos.go

@@ -37,7 +37,7 @@ func apiReposCreate(c *gin.Context) {
 		DefaultComponent    string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -51,7 +51,7 @@ func apiReposCreate(c *gin.Context) {
 
 	err := context.CollectionFactory().LocalRepoCollection().Add(repo)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -66,7 +66,7 @@ func apiReposEdit(c *gin.Context) {
 		DefaultComponent    *string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -76,7 +76,7 @@ func apiReposEdit(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
@@ -92,7 +92,7 @@ func apiReposEdit(c *gin.Context) {
 
 	err = collection.Update(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -107,7 +107,7 @@ func apiReposShow(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
@@ -132,27 +132,27 @@ func apiReposDrop(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	published := publishedCollection.ByLocalRepo(repo)
 	if len(published) > 0 {
-		c.Fail(409, fmt.Errorf("unable to drop, local repo is published"))
+		c.AbortWithError(409, fmt.Errorf("unable to drop, local repo is published"))
 		return
 	}
 
 	if !force {
 		snapshots := snapshotCollection.ByLocalRepoSource(repo)
 		if len(snapshots) > 0 {
-			c.Fail(409, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override"))
+			c.AbortWithError(409, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override"))
 			return
 		}
 	}
 
 	err = collection.Drop(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -167,13 +167,13 @@ func apiReposPackagesShow(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -186,7 +186,7 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 		PackageRefs []string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -196,19 +196,19 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -219,15 +219,15 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
 		if err != nil {
 			if err == database.ErrNotFound {
-				c.Fail(404, fmt.Errorf("package %s: %s", ref, err))
+				c.AbortWithError(404, fmt.Errorf("package %s: %s", ref, err))
 			} else {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 			}
 			return
 		}
 		err = cb(list, p)
 		if err != nil {
-			c.Fail(400, err)
+			c.AbortWithError(400, err)
 			return
 		}
 	}
@@ -236,7 +236,7 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to save: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to save: %s", err))
 		return
 	}
 
@@ -276,7 +276,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 	fileParam := c.Params.ByName("file")
 	if fileParam != "" && !verifyPath(fileParam) {
-		c.Fail(400, fmt.Errorf("wrong file"))
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
 		return
 	}
 
@@ -286,13 +286,13 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -301,6 +301,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 	var (
 		sources                      []string
 		packageFiles, failedFiles    []string
+		otherFiles                   []string
 		processedFiles, failedFiles2 []string
 		reporter                     = &aptly.RecordingResultReporter{
 			Warnings:     []string{},
@@ -316,11 +317,11 @@ func apiReposPackageFromDir(c *gin.Context) {
 		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
 	}
 
-	packageFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
+	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
 
 	list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to load packages: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to load packages: %s", err))
 		return
 	}
 
@@ -328,8 +329,10 @@ func apiReposPackageFromDir(c *gin.Context) {
 		context.CollectionFactory().PackageCollection(), reporter, nil, context.CollectionFactory().ChecksumCollection())
 	failedFiles = append(failedFiles, failedFiles2...)
 
+	processedFiles = append(processedFiles, otherFiles...)
+
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to import package files: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to import package files: %s", err))
 		return
 	}
 
@@ -337,7 +340,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to save: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to save: %s", err))
 		return
 	}
 

api/router.go

@@ -32,7 +32,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 
 			err = <-errCh
 			if err != nil {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 				return
 			}
 
@@ -40,7 +40,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 				requests <- dbRequest{releasedb, errCh}
 				err = <-errCh
 				if err != nil {
-					c.Fail(500, err)
+					c.AbortWithError(500, err)
 				}
 			}()
 

api/snapshot.go

@@ -42,7 +42,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 		Description string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -56,25 +56,25 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 
 	repo, err = collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = repo.CheckLock()
 	if err != nil {
-		c.Fail(409, err)
+		c.AbortWithError(409, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -84,7 +84,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 
 	err = snapshotCollection.Add(snapshot)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -105,7 +105,7 @@ func apiSnapshotsCreate(c *gin.Context) {
 		PackageRefs     []string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -124,13 +124,13 @@ func apiSnapshotsCreate(c *gin.Context) {
 	for i := range b.SourceSnapshots {
 		sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
 		if err != nil {
-			c.Fail(404, err)
+			c.AbortWithError(404, err)
 			return
 		}
 
 		err = snapshotCollection.LoadComplete(sources[i])
 		if err != nil {
-			c.Fail(500, err)
+			c.AbortWithError(500, err)
 			return
 		}
 	}
@@ -144,15 +144,15 @@ func apiSnapshotsCreate(c *gin.Context) {
 		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
 		if err != nil {
 			if err == database.ErrNotFound {
-				c.Fail(404, fmt.Errorf("package %s: %s", ref, err))
+				c.AbortWithError(404, fmt.Errorf("package %s: %s", ref, err))
 			} else {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 			}
 			return
 		}
 		err = list.Add(p)
 		if err != nil {
-			c.Fail(400, err)
+			c.AbortWithError(400, err)
 			return
 		}
 	}
@@ -161,7 +161,7 @@ func apiSnapshotsCreate(c *gin.Context) {
 
 	err = snapshotCollection.Add(snapshot)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -181,7 +181,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 		Description string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -195,19 +195,19 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 
 	repo, err = collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -217,7 +217,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 
 	err = snapshotCollection.Add(snapshot)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -236,7 +236,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 		Description string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -246,13 +246,13 @@ func apiSnapshotsUpdate(c *gin.Context) {
 
 	snapshot, err = collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	_, err = collection.ByName(b.Name)
 	if err == nil {
-		c.Fail(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
+		c.AbortWithError(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
 		return
 	}
 
@@ -266,7 +266,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 
 	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -281,13 +281,13 @@ func apiSnapshotsShow(c *gin.Context) {
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshot)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -309,28 +309,28 @@ func apiSnapshotsDrop(c *gin.Context) {
 
 	snapshot, err := snapshotCollection.ByName(name)
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	published := publishedCollection.BySnapshot(snapshot)
 
 	if len(published) > 0 {
-		c.Fail(409, fmt.Errorf("unable to drop: snapshot is published"))
+		c.AbortWithError(409, fmt.Errorf("unable to drop: snapshot is published"))
 		return
 	}
 
 	if !force {
 		snapshots := snapshotCollection.BySnapshotSource(snapshot)
 		if len(snapshots) > 0 {
-			c.Fail(409, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override"))
+			c.AbortWithError(409, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override"))
 			return
 		}
 	}
 
 	err = snapshotCollection.Drop(snapshot)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -347,32 +347,32 @@ func apiSnapshotsDiff(c *gin.Context) {
 
 	snapshotA, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshotA)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshotB)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	// Calculate diff
 	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -397,13 +397,13 @@ func apiSnapshotsSearchPackages(c *gin.Context) {
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshot)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 

aptly/interfaces.go

@@ -3,6 +3,7 @@
 package aptly
 
 import (
+	"context"
 	"io"
 	"os"
 
@@ -73,6 +74,14 @@ type PublishedStorage interface {
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file
 	RenameFile(oldName, newName string) error
+	// SymLink creates a symbolic link, which can be read with ReadLink
+	SymLink(src string, dst string) error
+	// HardLink creates a hardlink of a file
+	HardLink(src string, dst string) error
+	// FileExists returns true if path exists
+	FileExists(path string) (bool, error)
+	// ReadLink returns the symbolic link pointed to by path
+	ReadLink(path string) (string, error)
 }
 
 // FileSystemPublishedStorage is published storage on filesystem
@@ -116,9 +125,9 @@ type Progress interface {
 // Downloader is parallel HTTP fetcher
 type Downloader interface {
 	// Download starts new download task
-	Download(url string, destination string) error
+	Download(ctx context.Context, url string, destination string) error
 	// DownloadWithChecksum starts new download task with checksum verification
-	DownloadWithChecksum(url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error
+	DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error
 	// GetProgress returns Progress object
 	GetProgress() Progress
 }

bash_completion.d/aptly

@@ -164,7 +164,7 @@ _aptly()
           "edit")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -with-sources -with-udebs" -- ${cur}))
+                COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
               fi
@@ -499,7 +499,7 @@ _aptly()
           "snapshot"|"repo")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -distribution= -component= -gpg-key= -keyring= -label= -origin= -notautomatic= -butautomaticupgrades= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
               else
                 if [[ "$subcmd" == "snapshot" ]]; then
                   COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
@@ -524,7 +524,7 @@ _aptly()
           "update")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
               fi
@@ -539,7 +539,7 @@ _aptly()
           "switch")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
               fi
@@ -559,7 +559,7 @@ _aptly()
           "drop")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-force-drop" -- ${cur}))
+                COMPREPLY=($(compgen -W "-force-drop -skip-cleanup" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
               fi

cmd/context.go

@@ -29,3 +29,8 @@ func InitContext(flags *flag.FlagSet) error {
 
 	return err
 }
+
+// GetContext gives access to the context
+func GetContext() *ctx.AptlyContext {
+	return context
+}

cmd/mirror_edit.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -25,6 +26,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
+	fetchMirror := false
 	context.Flags().Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
@@ -35,6 +37,9 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 			repo.DownloadSources = flag.Value.Get().(bool)
 		case "with-udebs":
 			repo.DownloadUdebs = flag.Value.Get().(bool)
+		case "archive-url":
+			repo.SetArchiveRoot(flag.Value.String())
+			fetchMirror = true
 		}
 	})
 
@@ -51,8 +56,17 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 
 	if context.GlobalFlags().Lookup("architectures").Value.String() != "" {
 		repo.Architectures = context.ArchitecturesList()
+		fetchMirror = true
+	}
 
-		err = repo.Fetch(context.Downloader(), nil)
+	if fetchMirror {
+		var verifier pgp.Verifier
+		verifier, err = getVerifier(context.Flags())
+		if err != nil {
+			return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+		}
+
+		err = repo.Fetch(context.Downloader(), verifier)
 		if err != nil {
 			return fmt.Errorf("unable to edit: %s", err)
 		}
@@ -83,10 +97,13 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-edit", flag.ExitOnError),
 	}
 
+	cmd.Flag.String("archive-url", "", "archive url is the root of archive")
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd
 }

cmd/mirror_update.go

@@ -2,8 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"os"
-	"os/signal"
 	"strings"
 	"sync"
 
@@ -113,17 +111,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	// Catch ^C
-	sigch := make(chan os.Signal)
-	signal.Notify(sigch, os.Interrupt)
-	defer signal.Stop(sigch)
-
-	abort := make(chan struct{})
-	go func() {
-		<-sigch
-		signal.Stop(sigch)
-		close(abort)
-	}()
+	context.GoContextHandleSignals()
 
 	count := len(queue)
 	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
@@ -148,7 +136,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		for idx := range queue {
 			select {
 			case downloadQueue <- idx:
-			case <-abort:
+			case <-context.Done():
 				return
 			}
 		}
@@ -181,6 +169,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 
 					// download file...
 					e = context.Downloader().DownloadWithChecksum(
+						context,
 						repo.PackageURL(task.File.DownloadURL()).String(),
 						task.TempDownPath,
 						&task.File.Checksums,
@@ -190,28 +179,20 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 						pushError(e)
 						continue
 					}
-				case <-abort:
+
+					task.Done = true
+				case <-context.Done():
 					return
 				}
 			}
 		}()
 	}
 
-	// Wait for all downloads to finish
+	// Wait for all download goroutines to finish
 	wg.Wait()
 
-	select {
-	case <-abort:
-		return fmt.Errorf("unable to update: interrupted")
-	default:
-	}
-
 	context.Progress().ShutdownBar()
 
-	if len(errors) > 0 {
-		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
-	}
-
 	err = context.ReOpenDatabase()
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -221,11 +202,15 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	context.Progress().InitBar(int64(len(queue)), false)
 
 	for idx := range queue {
-
 		context.Progress().AddBar(1)
 
 		task := &queue[idx]
 
+		if !task.Done {
+			// download not finished yet
+			continue
+		}
+
 		// and import it back to the pool
 		task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, context.CollectionFactory().ChecksumCollection())
 		if err != nil {
@@ -237,16 +222,20 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 			additionalTask.File.PoolPath = task.File.PoolPath
 			additionalTask.File.Checksums = task.File.Checksums
 		}
-
-		select {
-		case <-abort:
-			return fmt.Errorf("unable to update: interrupted")
-		default:
-		}
 	}
 
 	context.Progress().ShutdownBar()
 
+	select {
+	case <-context.Done():
+		return fmt.Errorf("unable to update: interrupted")
+	default:
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
+	}
+
 	repo.FinalizeDownload(context.CollectionFactory(), context.Progress())
 	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
 	if err != nil {

cmd/publish_drop.go

@@ -24,7 +24,9 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 	storage, prefix := deb.ParsePrefix(param)
 
 	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress(), context.Flags().Lookup("force-drop").Value.Get().(bool))
+		context.CollectionFactory(), context.Progress(),
+		context.Flags().Lookup("force-drop").Value.Get().(bool),
+		context.Flags().Lookup("skip-cleanup").Value.Get().(bool))
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -50,6 +52,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/publish_repo.go

@@ -47,6 +47,7 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

cmd/publish_snapshot.go

@@ -137,6 +137,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("acquire-by-hash") {
+		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
+	}
+
 	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
 		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
@@ -227,6 +231,7 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

cmd/publish_switch.go

@@ -105,10 +105,13 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+			context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
 	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
@@ -151,6 +154,7 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/publish_update.go

@@ -69,10 +69,13 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+			context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
 	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
@@ -109,6 +112,7 @@ Example:
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/repo_add.go

@@ -41,9 +41,9 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 
-	var packageFiles, failedFiles []string
+	var packageFiles, otherFiles, failedFiles []string
 
-	packageFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
+	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
 
 	var processedFiles, failedFiles2 []string
 
@@ -55,6 +55,8 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to import package files: %s", err)
 	}
 
+	processedFiles = append(processedFiles, otherFiles...)
+
 	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)

cmd/repo_include.go

@@ -138,7 +138,7 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to load packages: %s", err)
 		}
 
-		packageFiles, _ := deb.CollectPackageFiles([]string{changes.TempDir}, reporter)
+		packageFiles, otherFiles, _ := deb.CollectPackageFiles([]string{changes.TempDir}, reporter)
 
 		var restriction deb.PackageQuery
 
@@ -179,6 +179,10 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
 		}
 
+		for _, file := range otherFiles {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
 		processedFiles = append(processedFiles, path)
 	}
 

context/context.go

@@ -2,9 +2,11 @@
 package context
 
 import (
+	gocontext "context"
 	"fmt"
 	"math/rand"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"runtime"
 	"runtime/pprof"
@@ -30,6 +32,8 @@ import (
 type AptlyContext struct {
 	sync.Mutex
 
+	gocontext.Context
+
 	flags, globalFlags *flag.FlagSet
 	configLoaded       bool
 
@@ -438,6 +442,27 @@ func (context *AptlyContext) GlobalFlags() *flag.FlagSet {
 	return context.globalFlags
 }
 
+// GoContextHandleSignals upgrades context to handle ^C by aborting context
+func (context *AptlyContext) GoContextHandleSignals() {
+	context.Lock()
+	defer context.Unlock()
+
+	// Catch ^C
+	sigch := make(chan os.Signal)
+	signal.Notify(sigch, os.Interrupt)
+
+	var cancel gocontext.CancelFunc
+
+	context.Context, cancel = gocontext.WithCancel(context.Context)
+
+	go func() {
+		<-sigch
+		signal.Stop(sigch)
+		context.Progress().PrintfStdErr("Aborting... press ^C once again to abort immediately\n")
+		cancel()
+	}()
+}
+
 // Shutdown shuts context down
 func (context *AptlyContext) Shutdown() {
 	context.Lock()
@@ -494,6 +519,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
 		flags:             flags,
 		globalFlags:       flags,
 		dependencyOptions: -1,
+		Context:           gocontext.TODO(),
 		publishedStorages: map[string]aptly.PublishedStorage{},
 	}
 

deb/deb.go

@@ -37,21 +37,46 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	library := ar.NewReader(file)
 	for {
 		header, err := library.Next()
+
 		if err == io.EOF {
-			return nil, fmt.Errorf("unable to find control.tar.gz part in package %s", packageFile)
+			return nil, fmt.Errorf("unable to find control.tar.* part in package %s", packageFile)
 		}
 		if err != nil {
 			return nil, fmt.Errorf("unable to read .deb archive %s: %s", packageFile, err)
 		}
 
-		if header.Name == "control.tar.gz" {
-			ungzip, err := gzip.NewReader(library)
-			if err != nil {
-				return nil, fmt.Errorf("unable to ungzip control file from %s. Error: %s", packageFile, err)
-			}
-			defer ungzip.Close()
+		// As per deb(5) version 1.19.0.4 the control file may be:
+		// - control.tar (since 1.17.6)
+		// - control.tar.gz
+		// - control.tar.xz (since 1.17.6)
+		// Look for all of the above and uncompress as necessary.
+		if strings.HasPrefix(header.Name, "control.tar") {
+			bufReader := bufio.NewReader(library)
 
-			untar := tar.NewReader(ungzip)
+			var tarInput io.Reader
+
+			switch header.Name {
+			case "control.tar":
+				tarInput = bufReader
+			case "control.tar.gz":
+				ungzip, err := gzip.NewReader(bufReader)
+				if err != nil {
+					return nil, errors.Wrapf(err, "unable to ungzip %s from %s", header.Name, packageFile)
+				}
+				defer ungzip.Close()
+				tarInput = ungzip
+			case "control.tar.xz":
+				unxz, err := xz.NewReader(bufReader)
+				if err != nil {
+					return nil, errors.Wrapf(err, "unable to unxz %s from %s", header.Name, packageFile)
+				}
+				defer unxz.Close()
+				tarInput = unxz
+			default:
+				return nil, fmt.Errorf("unsupported tar compression in %s: %s", packageFile, header.Name)
+			}
+
+			untar := tar.NewReader(tarInput)
 			for {
 				tarHeader, err := untar.Next()
 				if err == io.EOF {

deb/deb_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 type DebSuite struct {
-	debFile, debFile2, dscFile, dscFileNoSign string
+	debFile, debFile2, debFileWithXzControl, dscFile, dscFileNoSign string
 }
 
 var _ = Suite(&DebSuite{})
@@ -20,6 +20,7 @@ func (s *DebSuite) SetUpSuite(c *C) {
 	_, _File, _, _ := runtime.Caller(0)
 	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
 	s.debFile2 = filepath.Join(filepath.Dir(_File), "../system/changes/hardlink_0.2.1_amd64.deb")
+	s.debFileWithXzControl = filepath.Join(filepath.Dir(_File), "../system/changes/libqt5concurrent5-dbgsym_5.9.1+dfsg-2+18.04+bionic+build4_amd64.ddeb")
 	s.dscFile = filepath.Join(filepath.Dir(_File), "../system/files/pyspi_0.6.1-1.3.dsc")
 	s.dscFileNoSign = filepath.Join(filepath.Dir(_File), "../system/files/pyspi-0.6.1-1.3.stripped.dsc")
 }
@@ -38,6 +39,14 @@ func (s *DebSuite) TestGetControlFileFromDeb(c *C) {
 	c.Check(st["Package"], Equals, "libboost-program-options-dev")
 }
 
+func (s *DebSuite) TestGetControlFileFromDebWithXzControl(c *C) {
+	// Has control.tar.xz archive inside.
+	st, err := GetControlFileFromDeb(s.debFileWithXzControl)
+	c.Check(err, IsNil)
+	c.Check(st["Version"], Equals, "5.9.1+dfsg-2+18.04+bionic+build4")
+	c.Check(st["Package"], Equals, "libqt5concurrent5-dbgsym")
+}
+
 func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
 	verifier := &pgp.GoVerifier{}
 

deb/import.go

@@ -12,7 +12,7 @@ import (
 )
 
 // CollectPackageFiles walks filesystem collecting all candidates for package files
-func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, failedFiles []string) {
+func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, otherFiles, failedFiles []string) {
 	for _, location := range locations {
 		info, err2 := os.Stat(location)
 		if err2 != nil {
@@ -32,6 +32,8 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
 					strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
 					packageFiles = append(packageFiles, path)
+				} else if strings.HasSuffix(info.Name(), ".buildinfo") {
+					otherFiles = append(otherFiles, path)
 				}
 
 				return nil
@@ -46,6 +48,8 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
 				strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
 				packageFiles = append(packageFiles, location)
+			} else if strings.HasSuffix(info.Name(), ".buildinfo") {
+				otherFiles = append(otherFiles, location)
 			} else {
 				reporter.Warning("Unknown file extension: %s", location)
 				failedFiles = append(failedFiles, location)
@@ -147,14 +151,34 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 		// go over all the other files
 		for i := range files {
 			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(files[i].Filename))
-			files[i].PoolPath, err = pool.Import(sourceFile, files[i].Filename, &files[i].Checksums, false, checksumStorage)
+
+			_, err = os.Stat(sourceFile)
+			if err == nil {
+				files[i].PoolPath, err = pool.Import(sourceFile, files[i].Filename, &files[i].Checksums, false, checksumStorage)
+				if err == nil {
+					candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
+				}
+			} else if os.IsNotExist(err) {
+				// if file is not present, try to find it in the pool
+				var (
+					err2  error
+					found bool
+				)
+
+				files[i].PoolPath, found, err2 = pool.Verify("", files[i].Filename, &files[i].Checksums, checksumStorage)
+				if err2 != nil {
+					err = err2
+				} else if found {
+					// clear error, file is already in the package pool
+					err = nil
+				}
+			}
+
 			if err != nil {
 				reporter.Warning("Unable to import file %s into pool: %s", sourceFile, err)
 				failedFiles = append(failedFiles, file)
 				break
 			}
-
-			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
 		}
 		if err != nil {
 			// some files haven't been imported

deb/index_files.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -20,18 +21,20 @@ type indexFiles struct {
 	tempDir          string
 	suffix           string
 	indexes          map[string]*indexFile
+	acquireByHash    bool
 }
 
 type indexFile struct {
-	parent       *indexFiles
-	discardable  bool
-	compressable bool
-	onlyGzip     bool
-	signable     bool
-	relativePath string
-	tempFilename string
-	tempFile     *os.File
-	w            *bufio.Writer
+	parent        *indexFiles
+	discardable   bool
+	compressable  bool
+	onlyGzip      bool
+	signable      bool
+	acquireByHash bool
+	relativePath  string
+	tempFilename  string
+	tempFile      *os.File
+	w             *bufio.Writer
 }
 
 func (file *indexFile) BufWriter() (*bufio.Writer, error) {
@@ -91,11 +94,22 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 		file.parent.generatedFiles[file.relativePath+ext] = checksumInfo
 	}
 
-	err = file.parent.publishedStorage.MkDir(filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)))
+	filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
+
+	err = file.parent.publishedStorage.MkDir(filedir)
 	if err != nil {
 		return fmt.Errorf("unable to create dir: %s", err)
 	}
 
+	if file.acquireByHash {
+		for _, hash := range []string{"MD5Sum", "SHA1", "SHA256", "SHA512"} {
+			err = file.parent.publishedStorage.MkDir(filepath.Join(filedir, "by-hash", hash))
+			if err != nil {
+				return fmt.Errorf("unable to create dir: %s", err)
+			}
+		}
+	}
+
 	for _, ext := range exts {
 		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext),
 			file.tempFilename+ext)
@@ -107,6 +121,16 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] =
 				filepath.Join(file.parent.basePath, file.relativePath+ext)
 		}
+
+		if file.acquireByHash {
+			sums := file.parent.generatedFiles[file.relativePath+ext]
+			for hash, sum := range map[string]string{"SHA512": sums.SHA512, "SHA256": sums.SHA256, "SHA1": sums.SHA1, "MD5Sum": sums.MD5} {
+				err = packageIndexByHash(file, ext, hash, sum)
+				if err != nil {
+					return fmt.Errorf("unable to build hash file: %s", err)
+				}
+			}
+		}
 	}
 
 	if file.signable && signer != nil {
@@ -143,7 +167,53 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 	return nil
 }
 
-func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string) *indexFiles {
+func packageIndexByHash(file *indexFile, ext string, hash string, sum string) error {
+	src := filepath.Join(file.parent.basePath, file.relativePath)
+	indexfile := path.Base(src + ext)
+	src = src + file.parent.suffix + ext
+	filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
+	dst := filepath.Join(filedir, "by-hash", hash)
+	sumfilePath := filepath.Join(dst, sum)
+
+	// link already exists? do nothing
+	exists, err := file.parent.publishedStorage.FileExists(sumfilePath)
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error checking exists of file %s: %s", sumfilePath, err)
+	}
+	if exists {
+		return nil
+	}
+
+	// create the link
+	err = file.parent.publishedStorage.HardLink(src, sumfilePath)
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error creating hardlink %s: %s", sumfilePath, err)
+	}
+
+	// if a previous index file already exists exists, backup symlink
+	if exists, _ = file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile)); exists {
+		// if exists, remove old symlink
+		if exists, _ = file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile+".old")); exists {
+			var link string
+			link, err = file.parent.publishedStorage.ReadLink(filepath.Join(dst, indexfile+".old"))
+			if err != nil {
+				file.parent.publishedStorage.Remove(link)
+			}
+			file.parent.publishedStorage.Remove(filepath.Join(dst, indexfile+".old"))
+		}
+		file.parent.publishedStorage.RenameFile(filepath.Join(dst, indexfile),
+			filepath.Join(dst, indexfile+".old"))
+	}
+
+	// create symlink
+	err = file.parent.publishedStorage.SymLink(filepath.Join(dst, sum), filepath.Join(dst, indexfile))
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error creating symlink %s: %s", filepath.Join(dst, indexfile), err)
+	}
+	return nil
+}
+
+func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool) *indexFiles {
 	return &indexFiles{
 		publishedStorage: publishedStorage,
 		basePath:         basePath,
@@ -152,6 +222,7 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 		tempDir:          tempDir,
 		suffix:           suffix,
 		indexes:          make(map[string]*indexFile),
+		acquireByHash:    acquireByHash,
 	}
 }
 
@@ -175,11 +246,12 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 		}
 
 		file = &indexFile{
-			parent:       files,
-			discardable:  false,
-			compressable: true,
-			signable:     false,
-			relativePath: relativePath,
+			parent:        files,
+			discardable:   false,
+			compressable:  true,
+			signable:      false,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file
@@ -208,11 +280,12 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 		}
 
 		file = &indexFile{
-			parent:       files,
-			discardable:  udeb,
-			compressable: false,
-			signable:     false,
-			relativePath: relativePath,
+			parent:        files,
+			discardable:   udeb,
+			compressable:  false,
+			signable:      false,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file
@@ -237,12 +310,13 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index
 		}
 
 		file = &indexFile{
-			parent:       files,
-			discardable:  true,
-			compressable: true,
-			onlyGzip:     true,
-			signable:     false,
-			relativePath: relativePath,
+			parent:        files,
+			discardable:   true,
+			compressable:  true,
+			onlyGzip:      true,
+			signable:      false,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file

deb/list.go

@@ -124,6 +124,14 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 	return result, nil
 }
 
+// Has checks whether package is already in the list
+func (l *PackageList) Has(p *Package) bool {
+	key := l.keyFunc(p)
+	_, ok := l.packages[key]
+
+	return ok
+}
+
 // Add appends package to package list, additionally checking for uniqueness
 func (l *PackageList) Add(p *Package) error {
 	key := l.keyFunc(p)
@@ -441,18 +449,6 @@ func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*
 		panic("list not indexed, can't search")
 	}
 
-	if dep.Relation == VersionDontCare {
-		for _, p := range l.providesIndex[dep.Pkg] {
-			if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
-				searchResults = append(searchResults, p)
-
-				if !allMatches {
-					break
-				}
-			}
-		}
-	}
-
 	i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= dep.Pkg })
 
 	for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
@@ -468,6 +464,18 @@ func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*
 		i++
 	}
 
+	if dep.Relation == VersionDontCare {
+		for _, p := range l.providesIndex[dep.Pkg] {
+			if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
+				searchResults = append(searchResults, p)
+
+				if !allMatches {
+					break
+				}
+			}
+		}
+	}
+
 	return
 }
 
@@ -511,15 +519,27 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie
 
 			// try to satisfy dependencies
 			for _, dep := range missing {
-				// dependency might have already been satisfied
-				// with packages already been added
-				if result.Search(dep, false) != nil {
-					continue
+				if dependencyOptions&DepFollowAllVariants == 0 {
+					// dependency might have already been satisfied
+					// with packages already been added
+					//
+					// when follow-all-variants is enabled, we need to try to expand anyway,
+					// as even if dependency is satisfied now, there might be other ways to satisfy dependency
+					if result.Search(dep, false) != nil {
+						if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
+							progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true))
+						}
+						continue
+					}
 				}
 
 				searchResults := l.Search(dep, true)
-				if searchResults != nil {
+				if len(searchResults) > 0 {
 					for _, p := range searchResults {
+						if result.Has(p) {
+							continue
+						}
+
 						if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
 							progress.ColoredPrintf("@{g}Injecting package@|: %s", p)
 						}

deb/package.go

@@ -622,6 +622,7 @@ type PackageDownloadTask struct {
 	File         *PackageFile
 	Additional   []PackageDownloadTask
 	TempDownPath string
+	Done         bool
 }
 
 // DownloadList returns list of missing package files for download in format

deb/package_collection.go

@@ -315,7 +315,7 @@ func (collection *PackageCollection) SearchSupported() bool {
 
 // SearchByKey finds package by exact key
 func (collection *PackageCollection) SearchByKey(arch, name, version string) (result *PackageList) {
-	result = NewPackageList()
+	result = NewPackageListWithDuplicates(true, 0)
 
 	for _, key := range collection.db.KeysByPrefix([]byte(fmt.Sprintf("P%s %s %s", arch, name, version))) {
 		pkg, err := collection.ByKey(key)

deb/publish.go

@@ -64,6 +64,9 @@ type PublishedRepo struct {
 
 	// True if repo is being re-published
 	rePublishing bool
+
+	// Provide index files per hash also
+	AcquireByHash bool
 }
 
 // ParsePrefix splits [storage:]prefix into components
@@ -556,7 +559,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	}
 	defer os.RemoveAll(tempDir)
 
-	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix)
+	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash)
 
 	for component, list := range lists {
 		hadUdebs := false
@@ -683,6 +686,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				release["Component"] = component
 				release["Origin"] = p.GetOrigin()
 				release["Label"] = p.GetLabel()
+				if p.AcquireByHash {
+					release["Acquire-By-Hash"] = "yes"
+				}
 
 				var bufWriter *bufio.Writer
 				bufWriter, err = indexes.ReleaseIndex(component, arch, udeb).BufWriter()
@@ -720,6 +726,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	release["Codename"] = p.Distribution
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
+	if p.AcquireByHash {
+		release["Acquire-By-Hash"] = "yes"
+	}
 	release["Description"] = " Generated by aptly\n"
 	release["MD5Sum"] = ""
 	release["SHA1"] = ""
@@ -1097,7 +1106,7 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
 // Remove removes published repository, cleaning up directories, files
 func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly.PublishedStorageProvider,
 	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress,
-	force bool) error {
+	force, skipCleanup bool) error {
 	repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return err
@@ -1134,7 +1143,7 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
 		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
-	if len(cleanComponents) > 0 {
+	if !skipCleanup && len(cleanComponents) > 0 {
 		err = collection.CleanupPrefixComponentFiles(repo.Prefix, cleanComponents,
 			publishedStorageProvider.GetPublishedStorage(storage), collectionFactory, progress)
 		if err != nil {

deb/publish_test.go

@@ -756,7 +756,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
-	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
@@ -776,10 +776,48 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 
-	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, false)
 	c.Check(err, ErrorMatches, ".*not found")
 
-	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false)
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false, false)
+	c.Check(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2SkipCleanup(c *C) {
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, true)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	collection := NewPublishedRepoCollection(s.db)
+	_, err = collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, true)
+	c.Check(err, ErrorMatches, ".*not found")
+
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false, true)
 	c.Check(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
@@ -794,7 +832,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
-	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false)
+	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda")
@@ -816,7 +854,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) {
-	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false)
+	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")

deb/remote.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"bytes"
+	gocontext "context"
 	"fmt"
 	"log"
 	"net/url"
@@ -112,6 +113,12 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone
 	return result, nil
 }
 
+// SetArchiveRoot of remote repo
+func (repo *RemoteRepo) SetArchiveRoot(archiveRoot string) {
+	repo.ArchiveRoot = archiveRoot
+	repo.prepare()
+}
+
 func (repo *RemoteRepo) prepare() error {
 	var err error
 
@@ -252,13 +259,13 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier pgp.Verifier) error {
 
 	if verifier == nil {
 		// 0. Just download release file to temporary URL
-		release, err = http.DownloadTemp(d, repo.ReleaseURL("Release").String())
+		release, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("Release").String())
 		if err != nil {
 			return err
 		}
 	} else {
 		// 1. try InRelease file
-		inrelease, err = http.DownloadTemp(d, repo.ReleaseURL("InRelease").String())
+		inrelease, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("InRelease").String())
 		if err != nil {
 			goto splitsignature
 		}
@@ -280,12 +287,12 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier pgp.Verifier) error {
 
 	splitsignature:
 		// 2. try Release + Release.gpg
-		release, err = http.DownloadTemp(d, repo.ReleaseURL("Release").String())
+		release, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("Release").String())
 		if err != nil {
 			return err
 		}
 
-		releasesig, err = http.DownloadTemp(d, repo.ReleaseURL("Release.gpg").String())
+		releasesig, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("Release.gpg").String())
 		if err != nil {
 			return err
 		}
@@ -391,7 +398,10 @@ ok:
 		return err
 	}
 
-	delete(stanza, "SHA512")
+	err = parseSums("SHA512", func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data })
+	if err != nil {
+		return err
+	}
 
 	repo.Meta = stanza
 
@@ -430,7 +440,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 
 	for _, info := range packagesPaths {
 		path, kind := info[0], info[1]
-		packagesReader, packagesFile, err := http.DownloadTryCompression(d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries)
+		packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries)
 		if err != nil {
 			return err
 		}
@@ -592,7 +602,7 @@ func (repo *RemoteRepo) Decode(input []byte) error {
 	if err != nil {
 		if strings.HasPrefix(err.Error(), "codec.decoder: readContainerLen: Unrecognized descriptor byte: hex: 80") {
 			// probably it is broken DB from go < 1.2, try decoding w/o time.Time
-			var repo11 struct {
+			var repo11 struct { // nolint: maligned
 				UUID             string
 				Name             string
 				ArchiveRoot      string

deb/version.go

@@ -262,6 +262,13 @@ func ParseDependency(dep string) (d Dependency, err error) {
 	}
 
 	d.Pkg = strings.TrimSpace(dep[0:i])
+	if strings.ContainsRune(d.Pkg, ':') {
+		parts := strings.SplitN(d.Pkg, ":", 2)
+		d.Pkg, d.Architecture = parts[0], parts[1]
+		if d.Architecture == "any" {
+			d.Architecture = ""
+		}
+	}
 
 	rel := ""
 	if dep[i+1] == '>' || dep[i+1] == '<' || dep[i+1] == '=' {

deb/version_test.go

@@ -164,6 +164,20 @@ func (s *VersionSuite) TestParseDependency(c *C) {
 	c.Check(d.Version, Equals, "1.6")
 	c.Check(d.Architecture, Equals, "i386")
 
+	d, e = ParseDependency("python:any (>= 2.7~)")
+	c.Check(e, IsNil)
+	c.Check(d.Pkg, Equals, "python")
+	c.Check(d.Relation, Equals, VersionGreaterOrEqual)
+	c.Check(d.Version, Equals, "2.7~")
+	c.Check(d.Architecture, Equals, "")
+
+	d, e = ParseDependency("python:amd64 (>= 2.7~)")
+	c.Check(e, IsNil)
+	c.Check(d.Pkg, Equals, "python")
+	c.Check(d.Relation, Equals, VersionGreaterOrEqual)
+	c.Check(d.Version, Equals, "2.7~")
+	c.Check(d.Architecture, Equals, "amd64")
+
 	d, e = ParseDependency("dpkg{i386}")
 	c.Check(e, IsNil)
 	c.Check(d.Pkg, Equals, "dpkg")

files/public.go

@@ -247,3 +247,27 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
 	return os.Rename(filepath.Join(storage.rootPath, oldName), filepath.Join(storage.rootPath, newName))
 }
+
+// SymLink creates a symbolic link, which can be read with ReadLink
+func (storage *PublishedStorage) SymLink(src string, dst string) error {
+	return os.Symlink(filepath.Join(storage.rootPath, src), filepath.Join(storage.rootPath, dst))
+}
+
+// HardLink creates a hardlink of a file
+func (storage *PublishedStorage) HardLink(src string, dst string) error {
+	return os.Link(filepath.Join(storage.rootPath, src), filepath.Join(storage.rootPath, dst))
+}
+
+// FileExists returns true if path exists
+func (storage *PublishedStorage) FileExists(path string) (bool, error) {
+	if _, err := os.Lstat(filepath.Join(storage.rootPath, path)); os.IsNotExist(err) {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// ReadLink returns the symbolic link pointed to by path
+func (storage *PublishedStorage) ReadLink(path string) (string, error) {
+	return os.Readlink(path)
+}

files/public_test.go

@@ -103,6 +103,48 @@ func (s *PublishedStorageSuite) TestRenameFile(c *C) {
 	c.Assert(err, IsNil)
 }
 
+func (s *PublishedStorageSuite) TestFileExists(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	exists, _ := s.storage.FileExists("ppa/dists/squeeze/Release")
+	c.Check(exists, Equals, false)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	exists, _ = s.storage.FileExists("ppa/dists/squeeze/Release")
+	c.Check(exists, Equals, true)
+}
+
+func (s *PublishedStorageSuite) TestSymLink(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	err = s.storage.SymLink("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease")
+	c.Assert(err, IsNil)
+
+	exists, _ := s.storage.FileExists("ppa/dists/squeeze/InRelease")
+	c.Check(exists, Equals, true)
+}
+
+func (s *PublishedStorageSuite) TestHardLink(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	err = s.storage.HardLink("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease")
+	c.Assert(err, IsNil)
+
+	exists, _ := s.storage.FileExists("ppa/dists/squeeze/InRelease")
+	c.Check(exists, Equals, true)
+}
+
 func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)

http/compression.go

@@ -3,6 +3,7 @@ package http
 import (
 	"compress/bzip2"
 	"compress/gzip"
+	"context"
 	"fmt"
 	"io"
 	"net/url"
@@ -39,7 +40,7 @@ var compressionMethods = []struct {
 
 // DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until
 // it finds existing file.
-func DownloadTryCompression(downloader aptly.Downloader, baseURL *url.URL, path string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (io.Reader, *os.File, error) {
+func DownloadTryCompression(ctx context.Context, downloader aptly.Downloader, baseURL *url.URL, path string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (io.Reader, *os.File, error) {
 	var err error
 
 	for _, method := range compressionMethods {
@@ -63,13 +64,13 @@ func DownloadTryCompression(downloader aptly.Downloader, baseURL *url.URL, path
 
 		if foundChecksum {
 			expected := expectedChecksums[bestSuffix]
-			file, err = DownloadTempWithChecksum(downloader, tryURL.String(), &expected, ignoreMismatch, maxTries)
+			file, err = DownloadTempWithChecksum(ctx, downloader, tryURL.String(), &expected, ignoreMismatch, maxTries)
 		} else {
 			if !ignoreMismatch {
 				continue
 			}
 
-			file, err = DownloadTemp(downloader, tryURL.String())
+			file, err = DownloadTemp(ctx, downloader, tryURL.String())
 		}
 
 		if err != nil {

http/compression_test.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"context"
 	"errors"
 	"io"
 	"net/url"
@@ -12,6 +13,7 @@ import (
 
 type CompressionSuite struct {
 	baseURL *url.URL
+	ctx     context.Context
 }
 
 var _ = Suite(&CompressionSuite{})
@@ -25,6 +27,7 @@ const (
 
 func (s *CompressionSuite) SetUpTest(c *C) {
 	s.baseURL, _ = url.Parse("http://example.com/")
+	s.ctx = context.Background()
 }
 
 func (s *CompressionSuite) TestDownloadTryCompression(c *C) {
@@ -41,7 +44,7 @@ func (s *CompressionSuite) TestDownloadTryCompression(c *C) {
 	buf = make([]byte, 4)
 	d := NewFakeDownloader()
 	d.ExpectResponse("http://example.com/file.bz2", bzipData)
-	r, file, err := DownloadTryCompression(d, s.baseURL, "file", expectedChecksums, false, 1)
+	r, file, err := DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -53,7 +56,7 @@ func (s *CompressionSuite) TestDownloadTryCompression(c *C) {
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
 	d.ExpectResponse("http://example.com/file.gz", gzipData)
-	r, file, err = DownloadTryCompression(d, s.baseURL, "file", expectedChecksums, false, 1)
+	r, file, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -66,7 +69,7 @@ func (s *CompressionSuite) TestDownloadTryCompression(c *C) {
 	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
 	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
 	d.ExpectResponse("http://example.com/file.xz", xzData)
-	r, file, err = DownloadTryCompression(d, s.baseURL, "file", expectedChecksums, false, 1)
+	r, file, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -80,7 +83,7 @@ func (s *CompressionSuite) TestDownloadTryCompression(c *C) {
 	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
 	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
 	d.ExpectResponse("http://example.com/file", rawData)
-	r, file, err = DownloadTryCompression(d, s.baseURL, "file", expectedChecksums, false, 1)
+	r, file, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -91,7 +94,7 @@ func (s *CompressionSuite) TestDownloadTryCompression(c *C) {
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
 	d.ExpectResponse("http://example.com/file.gz", "x")
-	_, _, err = DownloadTryCompression(d, s.baseURL, "file", nil, true, 1)
+	_, _, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", nil, true, 1)
 	c.Assert(err, ErrorMatches, "unexpected EOF")
 	c.Assert(d.Empty(), Equals, true)
 }
@@ -109,7 +112,7 @@ func (s *CompressionSuite) TestDownloadTryCompressionLongestSuffix(c *C) {
 	buf = make([]byte, 4)
 	d := NewFakeDownloader()
 	d.ExpectResponse("http://example.com/subdir/file.bz2", bzipData)
-	r, file, err := DownloadTryCompression(d, s.baseURL, "subdir/file", expectedChecksums, false, 1)
+	r, file, err := DownloadTryCompression(s.ctx, d, s.baseURL, "subdir/file", expectedChecksums, false, 1)
 	c.Assert(err, IsNil)
 	defer file.Close()
 	io.ReadFull(r, buf)
@@ -119,7 +122,7 @@ func (s *CompressionSuite) TestDownloadTryCompressionLongestSuffix(c *C) {
 
 func (s *CompressionSuite) TestDownloadTryCompressionErrors(c *C) {
 	d := NewFakeDownloader()
-	_, _, err := DownloadTryCompression(d, s.baseURL, "file", nil, true, 1)
+	_, _, err := DownloadTryCompression(s.ctx, d, s.baseURL, "file", nil, true, 1)
 	c.Assert(err, ErrorMatches, "unexpected request.*")
 
 	d = NewFakeDownloader()
@@ -127,7 +130,7 @@ func (s *CompressionSuite) TestDownloadTryCompressionErrors(c *C) {
 	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
 	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
 	d.ExpectError("http://example.com/file", errors.New("403"))
-	_, _, err = DownloadTryCompression(d, s.baseURL, "file", nil, true, 1)
+	_, _, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", nil, true, 1)
 	c.Assert(err, ErrorMatches, "403")
 
 	d = NewFakeDownloader()
@@ -141,6 +144,6 @@ func (s *CompressionSuite) TestDownloadTryCompressionErrors(c *C) {
 		"file.xz":  {Size: 7},
 		"file":     {Size: 7},
 	}
-	_, _, err = DownloadTryCompression(d, s.baseURL, "file", expectedChecksums, false, 1)
+	_, _, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
 	c.Assert(err, ErrorMatches, "checksums don't match.*")
 }

http/download.go

@@ -1,12 +1,15 @@
 package http
 
 import (
+	"context"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/mxk/go-flowrate/flowrate"
@@ -62,12 +65,24 @@ func (downloader *downloaderImpl) GetProgress() aptly.Progress {
 }
 
 // Download starts new download task
-func (downloader *downloaderImpl) Download(url string, destination string) error {
-	return downloader.DownloadWithChecksum(url, destination, nil, false, 1)
+func (downloader *downloaderImpl) Download(ctx context.Context, url string, destination string) error {
+	return downloader.DownloadWithChecksum(ctx, url, destination, nil, false, 1)
+}
+
+func retryableError(err error) bool {
+	switch err.(type) {
+	case net.Error:
+		return true
+	case *net.OpError:
+		return true
+	case syscall.Errno:
+		return true
+	}
+	return false
 }
 
 // DownloadWithChecksum starts new download task with checksum verification
-func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string,
+func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url string, destination string,
 	expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
 
 	downloader.progress.Printf("Downloading %s...\n", url)
@@ -77,6 +92,7 @@ func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination s
 		return errors.Wrap(err, url)
 	}
 	req.Close = true
+	req = req.WithContext(ctx)
 
 	proxyURL, _ := downloader.client.Transport.(*http.Transport).Proxy(req)
 	if proxyURL == nil && (req.URL.Scheme == "http" || req.URL.Scheme == "https") {
@@ -88,10 +104,10 @@ func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination s
 	for maxTries > 0 {
 		temppath, err = downloader.download(req, url, destination, expected, ignoreMismatch)
 
-		if err != nil {
+		if err != nil && retryableError(err) {
 			maxTries--
 		} else {
-			// successful download
+			// get out of the loop
 			break
 		}
 	}

http/download_test.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -21,6 +22,7 @@ type DownloaderSuiteBase struct {
 	ch       chan struct{}
 	progress aptly.Progress
 	d        aptly.Downloader
+	ctx      context.Context
 }
 
 func (s *DownloaderSuiteBase) SetUpTest(c *C) {
@@ -44,6 +46,7 @@ func (s *DownloaderSuiteBase) SetUpTest(c *C) {
 	s.progress.Start()
 
 	s.d = NewDownloader(0, s.progress)
+	s.ctx = context.Background()
 }
 
 func (s *DownloaderSuiteBase) TearDownTest(c *C) {
@@ -71,52 +74,52 @@ func (s *DownloaderSuite) TearDownTest(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadOK(c *C) {
-	c.Assert(s.d.Download(s.url+"/test", s.tempfile.Name()), IsNil)
+	c.Assert(s.d.Download(s.ctx, s.url+"/test", s.tempfile.Name()), IsNil)
 }
 
 func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
-	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{}, false, 1),
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{}, false, 1),
 		ErrorMatches, ".*size check mismatch 12 != 0")
 
-	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, false, 1),
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, false, 1),
 		ErrorMatches, ".*md5 hash mismatch \"a1acb0fe91c7db45ec4d775192ec5738\" != \"abcdef\"")
 
-	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, true, 1),
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, true, 1),
 		IsNil)
 
-	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}, false, 1),
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}, false, 1),
 		IsNil)
 
-	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"}, false, 1),
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"}, false, 1),
 		ErrorMatches, ".*sha1 hash mismatch \"921893bae6ad6fd818401875d6779254ef0ff0ec\" != \"abcdef\"")
 
-	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
 		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"}, false, 1),
 		IsNil)
 
-	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
 		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"}, false, 1),
 		ErrorMatches, ".*sha256 hash mismatch \"b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac\" != \"abcdef\"")
 
 	checksums := utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
 		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}
-	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &checksums, false, 1),
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &checksums, false, 1),
 		IsNil)
 	// download backfills missing checksums
 	c.Check(checksums.SHA512, Equals, "bac18bf4e564856369acc2ed57300fecba3a2c1af5ae8304021e4252488678feb18118466382ee4e1210fe1f065080210e453a80cfb37ccb8752af3269df160e")
 }
 
 func (s *DownloaderSuite) TestDownload404(c *C) {
-	c.Assert(s.d.Download(s.url+"/doesntexist", s.tempfile.Name()),
+	c.Assert(s.d.Download(s.ctx, s.url+"/doesntexist", s.tempfile.Name()),
 		ErrorMatches, "HTTP code 404.*")
 }
 
 func (s *DownloaderSuite) TestDownloadConnectError(c *C) {
-	c.Assert(s.d.Download("http://nosuch.localhost/", s.tempfile.Name()),
+	c.Assert(s.d.Download(s.ctx, "http://nosuch.localhost/", s.tempfile.Name()),
 		ErrorMatches, ".*no such host")
 }
 
 func (s *DownloaderSuite) TestDownloadFileError(c *C) {
-	c.Assert(s.d.Download(s.url+"/test", "/"),
+	c.Assert(s.d.Download(s.ctx, s.url+"/test", "/"),
 		ErrorMatches, ".*permission denied")
 }

http/fake.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"os"
@@ -60,7 +61,7 @@ func (f *FakeDownloader) Empty() bool {
 }
 
 // DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification
-func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
+func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
 	var expectation expectedRequest
 	if len(f.expected) > 0 && f.expected[0].URL == url {
 		expectation, f.expected = f.expected[0], f.expected[1:]
@@ -109,8 +110,8 @@ func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, expec
 }
 
 // Download performs fake download by matching against first expectation in the queue
-func (f *FakeDownloader) Download(url string, filename string) error {
-	return f.DownloadWithChecksum(url, filename, nil, false, 1)
+func (f *FakeDownloader) Download(ctx context.Context, url string, filename string) error {
+	return f.DownloadWithChecksum(ctx, url, filename, nil, false, 1)
 }
 
 // GetProgress returns Progress object

http/temp.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"context"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -12,14 +13,14 @@ import (
 // DownloadTemp starts new download to temporary file and returns File
 //
 // Temporary file would be already removed, so no need to cleanup
-func DownloadTemp(downloader aptly.Downloader, url string) (*os.File, error) {
-	return DownloadTempWithChecksum(downloader, url, nil, false, 1)
+func DownloadTemp(ctx context.Context, downloader aptly.Downloader, url string) (*os.File, error) {
+	return DownloadTempWithChecksum(ctx, downloader, url, nil, false, 1)
 }
 
 // DownloadTempWithChecksum is a DownloadTemp with checksum verification
 //
 // Temporary file would be already removed, so no need to cleanup
-func DownloadTempWithChecksum(downloader aptly.Downloader, url string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (*os.File, error) {
+func DownloadTempWithChecksum(ctx context.Context, downloader aptly.Downloader, url string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (*os.File, error) {
 	tempdir, err := ioutil.TempDir(os.TempDir(), "aptly")
 	if err != nil {
 		return nil, err
@@ -33,7 +34,7 @@ func DownloadTempWithChecksum(downloader aptly.Downloader, url string, expected
 		defer downloader.GetProgress().ShutdownBar()
 	}
 
-	err = downloader.DownloadWithChecksum(url, tempfile, expected, ignoreMismatch, maxTries)
+	err = downloader.DownloadWithChecksum(ctx, url, tempfile, expected, ignoreMismatch, maxTries)
 	if err != nil {
 		return nil, err
 	}

http/temp_test.go

@@ -23,7 +23,7 @@ func (s *TempSuite) TearDownTest(c *C) {
 }
 
 func (s *TempSuite) TestDownloadTemp(c *C) {
-	f, err := DownloadTemp(s.d, s.url+"/test")
+	f, err := DownloadTemp(s.ctx, s.d, s.url+"/test")
 	c.Assert(err, IsNil)
 	defer f.Close()
 
@@ -37,18 +37,18 @@ func (s *TempSuite) TestDownloadTemp(c *C) {
 }
 
 func (s *TempSuite) TestDownloadTempWithChecksum(c *C) {
-	f, err := DownloadTempWithChecksum(s.d, s.url+"/test", &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+	f, err := DownloadTempWithChecksum(s.ctx, s.d, s.url+"/test", &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
 		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false, 1)
 	c.Assert(err, IsNil)
 
 	c.Assert(f.Close(), IsNil)
 
-	_, err = DownloadTempWithChecksum(s.d, s.url+"/test", &utils.ChecksumInfo{Size: 13}, false, 1)
+	_, err = DownloadTempWithChecksum(s.ctx, s.d, s.url+"/test", &utils.ChecksumInfo{Size: 13}, false, 1)
 	c.Assert(err, ErrorMatches, ".*size check mismatch 12 != 13")
 }
 
 func (s *TempSuite) TestDownloadTempError(c *C) {
-	f, err := DownloadTemp(s.d, s.url+"/doesntexist")
+	f, err := DownloadTemp(s.ctx, s.d, s.url+"/doesntexist")
 	c.Assert(err, NotNil)
 	c.Assert(f, IsNil)
 	c.Assert(err, ErrorMatches, "HTTP code 404.*")

linter.json

@@ -12,7 +12,7 @@
         "staticcheck",
         "varcheck",
         "structcheck",
-        "aligncheck",
+        "maligned",
         "vetshadow",
         "goconst",
         "interfacer"

man/aptly.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "APTLY" "1" "July 2017" "" ""
+.TH "APTLY" "1" "November 2017" "" ""
 .
 .SH "NAME"
 \fBaptly\fR \- Debian repository management tool
@@ -413,23 +413,23 @@ location of configuration file (default locations are /etc/aptly\.conf, ~/\.aptl
 number of attempts to open DB if it\(cqs locked by other instance
 .
 .TP
-\-\fBdep\-follow\-all\-variants\fR=false
+\-\fBdep\-follow\-all\-variants\fR
 when processing dependencies, follow a & b if dependency is \(cqa|b\(cq
 .
 .TP
-\-\fBdep\-follow\-recommends\fR=false
+\-\fBdep\-follow\-recommends\fR
 when processing dependencies, follow Recommends
 .
 .TP
-\-\fBdep\-follow\-source\fR=false
+\-\fBdep\-follow\-source\fR
 when processing dependencies, follow from binary to Source packages
 .
 .TP
-\-\fBdep\-follow\-suggests\fR=false
+\-\fBdep\-follow\-suggests\fR
 when processing dependencies, follow Suggests
 .
 .TP
-\-\fBdep\-verbose\-resolve\fR=false
+\-\fBdep\-verbose\-resolve\fR
 when processing dependencies, print detailed logs
 .
 .TP
@@ -462,19 +462,19 @@ Options:
 filter packages in mirror
 .
 .TP
-\-\fBfilter\-with\-deps\fR=false
+\-\fBfilter\-with\-deps\fR
 when filtering, include dependencies of matching packages as well
 .
 .TP
-\-\fBforce\-architectures\fR=false
+\-\fBforce\-architectures\fR
 (only with architecture list) skip check that requested architectures are listed in Release file
 .
 .TP
-\-\fBforce\-components\fR=false
+\-\fBforce\-components\fR
 (only with component list) skip check that requested components are listed in Release file
 .
 .TP
-\-\fBignore\-signatures\fR=false
+\-\fBignore\-signatures\fR
 disable verification of Release file signatures
 .
 .TP
@@ -482,11 +482,11 @@ disable verification of Release file signatures
 gpg keyring to use when verifying Release file (could be specified multiple times)
 .
 .TP
-\-\fBwith\-sources\fR=false
+\-\fBwith\-sources\fR
 download source packages in addition to binary packages
 .
 .TP
-\-\fBwith\-udebs\fR=false
+\-\fBwith\-udebs\fR
 download \.udeb packages (Debian installer support)
 .
 .SH "LIST MIRRORS"
@@ -505,7 +505,7 @@ $ aptly mirror list
 Options:
 .
 .TP
-\-\fBraw\fR=false
+\-\fBraw\fR
 display list in machine\-readable format
 .
 .SH "SHOW DETAILS ABOUT MIRROR"
@@ -524,7 +524,7 @@ $ aptly mirror show wheezy\-main
 Options:
 .
 .TP
-\-\fBwith\-packages\fR=false
+\-\fBwith\-packages\fR
 show detailed list of packages and versions stored in the mirror
 .
 .SH "DELETE MIRROR"
@@ -543,7 +543,7 @@ $ aptly mirror drop wheezy\-main
 Options:
 .
 .TP
-\-\fBforce\fR=false
+\-\fBforce\fR
 force mirror deletion even if used by snapshots
 .
 .SH "UPDATE MIRROR"
@@ -566,15 +566,15 @@ Options:
 limit download speed (kbytes/sec)
 .
 .TP
-\-\fBforce\fR=false
+\-\fBforce\fR
 force update mirror even if it is locked by another process
 .
 .TP
-\-\fBignore\-checksums\fR=false
+\-\fBignore\-checksums\fR
 ignore checksum mismatches while downloading package files and metadata
 .
 .TP
-\-\fBignore\-signatures\fR=false
+\-\fBignore\-signatures\fR
 disable verification of Release file signatures
 .
 .TP
@@ -586,7 +586,7 @@ gpg keyring to use when verifying Release file (could be specified multiple time
 max download tries till process fails with download error
 .
 .TP
-\-\fBskip\-existing\-packages\fR=false
+\-\fBskip\-existing\-packages\fR
 do not check file existence for packages listed in the internal database of the mirror
 .
 .SH "RENAMES MIRROR"
@@ -617,19 +617,31 @@ $ aptly mirror edit \-filter=nginx \-filter\-with\-deps some\-mirror
 Options:
 .
 .TP
+\-\fBarchive\-url\fR=
+archive url is the root of archive
+.
+.TP
 \-\fBfilter\fR=
 filter packages in mirror
 .
 .TP
-\-\fBfilter\-with\-deps\fR=false
+\-\fBfilter\-with\-deps\fR
 when filtering, include dependencies of matching packages as well
 .
 .TP
-\-\fBwith\-sources\fR=false
+\-\fBignore\-signatures\fR
+disable verification of Release file signatures
+.
+.TP
+\-\fBkeyring\fR=
+gpg keyring to use when verifying Release file (could be specified multiple times)
+.
+.TP
+\-\fBwith\-sources\fR
 download source packages in addition to binary packages
 .
 .TP
-\-\fBwith\-udebs\fR=false
+\-\fBwith\-udebs\fR
 download \.udeb packages (Debian installer support)
 .
 .SH "SEARCH MIRROR FOR PACKAGES MATCHING QUERY"
@@ -662,7 +674,7 @@ Options:
 custom format for result printing
 .
 .TP
-\-\fBwith\-deps\fR=false
+\-\fBwith\-deps\fR
 include dependencies into search results
 .
 .SH "ADD PACKAGES TO LOCAL REPOSITORY"
@@ -681,11 +693,11 @@ $ aptly repo add testing myapp\-0\.1\.2\.deb incoming/
 Options:
 .
 .TP
-\-\fBforce\-replace\fR=false
+\-\fBforce\-replace\fR
 when adding package that conflicts with existing package, remove existing package
 .
 .TP
-\-\fBremove\-files\fR=false
+\-\fBremove\-files\fR
 remove files that have been imported successfully into repository
 .
 .SH "COPY PACKAGES BETWEEN LOCAL REPOSITORIES"
@@ -704,11 +716,11 @@ $ aptly repo copy testing stable \(cqmyapp (=0\.1\.12)\(cq
 Options:
 .
 .TP
-\-\fBdry\-run\fR=false
+\-\fBdry\-run\fR
 don\(cqt copy, just show what would be copied
 .
 .TP
-\-\fBwith\-deps\fR=false
+\-\fBwith\-deps\fR
 follow dependencies when processing package\-spec
 .
 .SH "CREATE LOCAL REPOSITORY"
@@ -764,7 +776,7 @@ $ aptly repo drop local\-repo
 Options:
 .
 .TP
-\-\fBforce\fR=false
+\-\fBforce\fR
 force local repo deletion even if used by snapshots
 .
 .SH "EDIT PROPERTIES OF LOCAL REPOSITORY"
@@ -814,11 +826,11 @@ $ aptly repo import wheezy\-main testing nginx
 Options:
 .
 .TP
-\-\fBdry\-run\fR=false
+\-\fBdry\-run\fR
 don\(cqt import, just show what would be imported
 .
 .TP
-\-\fBwith\-deps\fR=false
+\-\fBwith\-deps\fR
 follow dependencies when processing package\-spec
 .
 .SH "LIST LOCAL REPOSITORIES"
@@ -837,7 +849,7 @@ $ aptly repo list
 Options:
 .
 .TP
-\-\fBraw\fR=false
+\-\fBraw\fR
 display list in machine\-readable format
 .
 .SH "MOVE PACKAGES BETWEEN LOCAL REPOSITORIES"
@@ -856,11 +868,11 @@ $ aptly repo move testing stable \(cqmyapp (=0\.1\.12)\(cq
 Options:
 .
 .TP
-\-\fBdry\-run\fR=false
+\-\fBdry\-run\fR
 don\(cqt move, just show what would be moved
 .
 .TP
-\-\fBwith\-deps\fR=false
+\-\fBwith\-deps\fR
 follow dependencies when processing package\-spec
 .
 .SH "REMOVE PACKAGES FROM LOCAL REPOSITORY"
@@ -879,7 +891,7 @@ $ aptly repo remove testing \(cqmyapp (=0\.1\.12)\(cq
 Options:
 .
 .TP
-\-\fBdry\-run\fR=false
+\-\fBdry\-run\fR
 don\(cqt remove, just show what would be removed
 .
 .SH "SHOW DETAILS ABOUT LOCAL REPOSITORY"
@@ -895,7 +907,7 @@ ex: $ aptly repo show testing
 Options:
 .
 .TP
-\-\fBwith\-packages\fR=false
+\-\fBwith\-packages\fR
 show list of packages
 .
 .SH "RENAMES LOCAL REPOSITORY"
@@ -940,7 +952,7 @@ Options:
 custom format for result printing
 .
 .TP
-\-\fBwith\-deps\fR=false
+\-\fBwith\-deps\fR
 include dependencies into search results
 .
 .SH "ADD PACKAGES TO LOCAL REPOSITORIES BASED ON \.CHANGES FILES"
@@ -962,15 +974,15 @@ $ aptly repo include \-repo=foo\-release incoming/
 Options:
 .
 .TP
-\-\fBaccept\-unsigned\fR=false
+\-\fBaccept\-unsigned\fR
 accept unsigned \.changes files
 .
 .TP
-\-\fBforce\-replace\fR=false
+\-\fBforce\-replace\fR
 when adding package that conflicts with existing package, remove existing package
 .
 .TP
-\-\fBignore\-signatures\fR=false
+\-\fBignore\-signatures\fR
 disable verification of \.changes file signature
 .
 .TP
@@ -978,7 +990,7 @@ disable verification of \.changes file signature
 gpg keyring to use when verifying Release file (could be specified multiple times)
 .
 .TP
-\-\fBno\-remove\-files\fR=false
+\-\fBno\-remove\-files\fR
 don\(cqt remove files that have been imported successfully into repository
 .
 .TP
@@ -1023,7 +1035,7 @@ $ aptly snapshot list
 Options:
 .
 .TP
-\-\fBraw\fR=false
+\-\fBraw\fR
 display list in machine\-readable format
 .
 .TP
@@ -1053,7 +1065,7 @@ $ aptly snapshot show wheezy\-main
 Options:
 .
 .TP
-\-\fBwith\-packages\fR=false
+\-\fBwith\-packages\fR
 show list of packages
 .
 .SH "VERIFY DEPENDENCIES IN SNAPSHOT"
@@ -1098,19 +1110,19 @@ $ aptly snapshot pull wheezy\-main wheezy\-backports wheezy\-new\-xorg xorg\-ser
 Options:
 .
 .TP
-\-\fBall\-matches\fR=false
+\-\fBall\-matches\fR
 pull all the packages that satisfy the dependency version requirements
 .
 .TP
-\-\fBdry\-run\fR=false
+\-\fBdry\-run\fR
 don\(cqt create destination snapshot, just show what would be pulled
 .
 .TP
-\-\fBno\-deps\fR=false
+\-\fBno\-deps\fR
 don\(cqt process dependencies, just pull listed packages
 .
 .TP
-\-\fBno\-remove\fR=false
+\-\fBno\-remove\fR
 don\(cqt remove other package versions when pulling package
 .
 .SH "DIFFERENCE BETWEEN TWO SNAPSHOTS"
@@ -1136,7 +1148,7 @@ $ aptly snapshot diff \-only\-matching wheezy\-main wheezy\-backports
 Options:
 .
 .TP
-\-\fBonly\-matching\fR=false
+\-\fBonly\-matching\fR
 display diff only for matching packages (don\(cqt display missing packages)
 .
 .SH "MERGES SNAPSHOTS"
@@ -1162,11 +1174,11 @@ $ aptly snapshot merge wheezy\-w\-backports wheezy\-main wheezy\-backports
 Options:
 .
 .TP
-\-\fBlatest\fR=false
+\-\fBlatest\fR
 use only the latest version of each package
 .
 .TP
-\-\fBno\-remove\fR=false
+\-\fBno\-remove\fR
 don\(cqt remove duplicate arch/name packages
 .
 .SH "DELETE SNAPSHOT"
@@ -1192,7 +1204,7 @@ $ aptly snapshot drop wheezy\-main
 Options:
 .
 .TP
-\-\fBforce\fR=false
+\-\fBforce\fR
 remove snapshot even if it was used as source for other snapshots
 .
 .SH "RENAMES SNAPSHOT"
@@ -1237,7 +1249,7 @@ Options:
 custom format for result printing
 .
 .TP
-\-\fBwith\-deps\fR=false
+\-\fBwith\-deps\fR
 include dependencies into search results
 .
 .SH "FILTER PACKAGES IN SNAPSHOT PRODUCING ANOTHER SNAPSHOT"
@@ -1263,7 +1275,7 @@ $ aptly snapshot filter wheezy\-main wheezy\-required \(cqPriorioty (required)\(
 Options:
 .
 .TP
-\-\fBwith\-deps\fR=false
+\-\fBwith\-deps\fR
 include dependent packages as well
 .
 .SH "REMOVE PUBLISHED REPOSITORY"
@@ -1289,9 +1301,13 @@ $ aptly publish drop wheezy
 Options:
 .
 .TP
-\-\fBforce\-drop\fR=false
+\-\fBforce\-drop\fR
 remove published repository even if some files could not be cleaned up
 .
+.TP
+\-\fBskip\-cleanup\fR
+don\(cqt remove unreferenced files in prefix/component
+.
 .SH "LIST OF PUBLISHED REPOSITORIES"
 \fBaptly\fR \fBpublish\fR \fBlist\fR
 .
@@ -1315,7 +1331,7 @@ $ aptly publish list
 Options:
 .
 .TP
-\-\fBraw\fR=false
+\-\fBraw\fR
 display list in machine\-readable format
 .
 .SH "PUBLISH LOCAL REPOSITORY"
@@ -1357,7 +1373,11 @@ $ aptly publish repo testing
 Options:
 .
 .TP
-\-\fBbatch\fR=false
+\-\fBacquire\-by\-hash\fR
+provide index files by hash
+.
+.TP
+\-\fBbatch\fR
 run GPG with detached tty
 .
 .TP
@@ -1373,7 +1393,7 @@ component name to publish (for multi\-component publishing, separate components
 distribution name to publish
 .
 .TP
-\-\fBforce\-overwrite\fR=false
+\-\fBforce\-overwrite\fR
 overwrite files in package pool in case of mismatch
 .
 .TP
@@ -1409,11 +1429,11 @@ GPG passhprase\-file for the key (warning: could be insecure)
 GPG secret keyring to use (instead of default)
 .
 .TP
-\-\fBskip\-contents\fR=false
+\-\fBskip\-contents\fR
 don\(cqt generate Contents indexes
 .
 .TP
-\-\fBskip\-signing\fR=false
+\-\fBskip\-signing\fR
 don\(cqt sign Release files with GPG
 .
 .SH "PUBLISH SNAPSHOT"
@@ -1452,7 +1472,11 @@ $ aptly publish snapshot wheezy\-main
 Options:
 .
 .TP
-\-\fBbatch\fR=false
+\-\fBacquire\-by\-hash\fR
+provide index files by hash
+.
+.TP
+\-\fBbatch\fR
 run GPG with detached tty
 .
 .TP
@@ -1468,7 +1492,7 @@ component name to publish (for multi\-component publishing, separate components
 distribution name to publish
 .
 .TP
-\-\fBforce\-overwrite\fR=false
+\-\fBforce\-overwrite\fR
 overwrite files in package pool in case of mismatch
 .
 .TP
@@ -1504,11 +1528,11 @@ GPG passhprase\-file for the key (warning: could be insecure)
 GPG secret keyring to use (instead of default)
 .
 .TP
-\-\fBskip\-contents\fR=false
+\-\fBskip\-contents\fR
 don\(cqt generate Contents indexes
 .
 .TP
-\-\fBskip\-signing\fR=false
+\-\fBskip\-signing\fR
 don\(cqt sign Release files with GPG
 .
 .SH "UPDATE PUBLISHED REPOSITORY BY SWITCHING TO NEW SNAPSHOT"
@@ -1550,7 +1574,7 @@ This command would switch published repository (with one component) named ppa/wh
 Options:
 .
 .TP
-\-\fBbatch\fR=false
+\-\fBbatch\fR
 run GPG with detached tty
 .
 .TP
@@ -1558,7 +1582,7 @@ run GPG with detached tty
 component names to update (for multi\-component publishing, separate components with commas)
 .
 .TP
-\-\fBforce\-overwrite\fR=false
+\-\fBforce\-overwrite\fR
 overwrite files in package pool in case of mismatch
 .
 .TP
@@ -1582,11 +1606,15 @@ GPG passhprase\-file for the key (warning: could be insecure)
 GPG secret keyring to use (instead of default)
 .
 .TP
-\-\fBskip\-contents\fR=false
+\-\fBskip\-cleanup\fR
+don\(cqt remove unreferenced files in prefix/component
+.
+.TP
+\-\fBskip\-contents\fR
 don\(cqt generate Contents indexes
 .
 .TP
-\-\fBskip\-signing\fR=false
+\-\fBskip\-signing\fR
 don\(cqt sign Release files with GPG
 .
 .SH "UPDATE PUBLISHED LOCAL REPOSITORY"
@@ -1615,11 +1643,11 @@ $ aptly publish update wheezy ppa
 Options:
 .
 .TP
-\-\fBbatch\fR=false
+\-\fBbatch\fR
 run GPG with detached tty
 .
 .TP
-\-\fBforce\-overwrite\fR=false
+\-\fBforce\-overwrite\fR
 overwrite files in package pool in case of mismatch
 .
 .TP
@@ -1643,11 +1671,15 @@ GPG passhprase\-file for the key (warning: could be insecure)
 GPG secret keyring to use (instead of default)
 .
 .TP
-\-\fBskip\-contents\fR=false
+\-\fBskip\-cleanup\fR
+don\(cqt remove unreferenced files in prefix/component
+.
+.TP
+\-\fBskip\-contents\fR
 don\(cqt generate Contents indexes
 .
 .TP
-\-\fBskip\-signing\fR=false
+\-\fBskip\-signing\fR
 don\(cqt sign Release files with GPG
 .
 .SH "SHOWS DETAILS OF PUBLISHED REPOSITORY"
@@ -1721,11 +1753,11 @@ $ aptly package show \(cqnginx\-light_1\.2\.1\-2\.2+wheezy2_i386\(cq
 Options:
 .
 .TP
-\-\fBwith\-files\fR=false
+\-\fBwith\-files\fR
 display information about files from package pool
 .
 .TP
-\-\fBwith\-references\fR=false
+\-\fBwith\-references\fR
 display information about mirrors, snapshots and local repos referencing this package
 .
 .SH "CLEANUP DB AND PACKAGE POOL"
@@ -1744,11 +1776,11 @@ $ aptly db cleanup
 Options:
 .
 .TP
-\-\fBdry\-run\fR=false
+\-\fBdry\-run\fR
 don\(cqt delete anything
 .
 .TP
-\-\fBverbose\fR=false
+\-\fBverbose\fR
 be verbose when loading objects/removing them
 .
 .SH "RECOVER DB AFTER CRASH"
@@ -1802,7 +1834,7 @@ Options:
 host:port for HTTP listening or unix://path to listen on a Unix domain socket
 .
 .TP
-\-\fBno\-lock\fR=false
+\-\fBno\-lock\fR
 don\(cqt lock the database
 .
 .SH "RENDER GRAPH OF RELATIONSHIPS"
@@ -1992,4 +2024,11 @@ Clemens Rabe (https://github\.com/seeraven)
 .IP "\[ci]" 4
 TJ Merritt (https://github\.com/tjmerritt)
 .
+.IP "\[ci]" 4
+Matt Martyn (https://github\.com/MMartyn)
+.
+.IP "\[ci]" 4
+Ludovico Cavedon (https://github\.com/cavedon)
+.
 .IP "" 0
+

man/aptly.1.ronn.tmpl

@@ -440,7 +440,7 @@ Options:
 {{/* options layout */}}
 {{define "options"}}
 {{range allFlags .Flag}}
-  * -`{{.Name}}`={{.DefValue}}:
+  * -`{{.Name}}`{{if ne .DefValue "false"}}={{.DefValue}}{{end}}:
     {{.Usage}}
 {{end}}
 {{end}}

query/syntax.go

@@ -135,7 +135,7 @@ func (p *parser) D() deb.PackageQuery {
 	operator, value := p.Condition()
 
 	r, _ := utf8.DecodeRuneInString(field)
-	if strings.HasPrefix(field, "$") || unicode.IsUpper(r) {
+	if strings.HasPrefix(field, "$") || (unicode.IsUpper(r) && !strings.ContainsRune(field, '_')) {
 		// special field or regular field
 		q := &deb.FieldQuery{Field: field, Relation: operatorToRelation(operator), Value: value}
 		if q.Relation == deb.VersionRegexp {

query/syntax_test.go

@@ -57,6 +57,18 @@ func (s *SyntaxSuite) TestParsing(c *C) {
 	c.Assert(err, IsNil)
 	c.Check(q, DeepEquals, &deb.PkgQuery{Pkg: "alien-data", Version: "1.3.4~dev", Arch: "i386"})
 
+	l, _ = lex("query", "Alien-data_1.3.4~dev_i386")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q, DeepEquals, &deb.PkgQuery{Pkg: "Alien-data", Version: "1.3.4~dev", Arch: "i386"})
+
+	l, _ = lex("query", "Name")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q, DeepEquals, &deb.FieldQuery{Field: "Name"})
+
 	l, _ = lex("query", "package (> 5.3.7) {amd64}")
 	q, err = parse(l)
 

s3/public.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/corehandlers"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/request"
@@ -183,7 +184,7 @@ func (storage *PublishedStorage) putFile(path string, source io.ReadSeeker) erro
 func (storage *PublishedStorage) Remove(path string) error {
 	params := &s3.DeleteObjectInput{
 		Bucket: aws.String(storage.bucket),
-		Key:    aws.String(path),
+		Key:    aws.String(filepath.Join(storage.prefix, path)),
 	}
 	_, err := storage.s3.DeleteObject(params)
 	if err != nil {
@@ -267,7 +268,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, baseName strin
 	poolPath := filepath.Join(storage.prefix, relPath)
 
 	if storage.pathCache == nil {
-		paths, md5s, err := storage.internalFilelist(storage.prefix, true)
+		paths, md5s, err := storage.internalFilelist("", true)
 		if err != nil {
 			return errors.Wrap(err, "error caching paths under prefix")
 		}
@@ -370,6 +371,13 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
 		ACL:        aws.String(storage.acl),
 	}
 
+	if storage.storageClass != "" {
+		params.StorageClass = aws.String(storage.storageClass)
+	}
+	if storage.encryptionMethod != "" {
+		params.ServerSideEncryption = aws.String(storage.encryptionMethod)
+	}
+
 	_, err := storage.s3.CopyObject(params)
 	if err != nil {
 		return fmt.Errorf("error copying %s -> %s in %s: %s", oldName, newName, storage, err)
@@ -377,3 +385,71 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
 
 	return storage.Remove(oldName)
 }
+
+// SymLink creates a copy of src file and adds link information as meta data
+func (storage *PublishedStorage) SymLink(src string, dst string) error {
+
+	params := &s3.CopyObjectInput{
+		Bucket:     aws.String(storage.bucket),
+		CopySource: aws.String(filepath.Join(storage.prefix, src)),
+		Key:        aws.String(filepath.Join(storage.prefix, dst)),
+		ACL:        aws.String(storage.acl),
+		Metadata: map[string]*string{
+			"SymLink": aws.String(src),
+		},
+		MetadataDirective: aws.String("REPLACE"),
+	}
+
+	if storage.storageClass != "" {
+		params.StorageClass = aws.String(storage.storageClass)
+	}
+	if storage.encryptionMethod != "" {
+		params.ServerSideEncryption = aws.String(storage.encryptionMethod)
+	}
+
+	_, err := storage.s3.CopyObject(params)
+	if err != nil {
+		return fmt.Errorf("error symlinking %s -> %s in %s: %s", src, dst, storage, err)
+	}
+
+	return err
+}
+
+// HardLink using symlink functionality as hard links do not exist
+func (storage *PublishedStorage) HardLink(src string, dst string) error {
+	return storage.SymLink(src, dst)
+}
+
+// FileExists returns true if path exists
+func (storage *PublishedStorage) FileExists(path string) (bool, error) {
+	params := &s3.HeadObjectInput{
+		Bucket: aws.String(storage.bucket),
+		Key:    aws.String(filepath.Join(storage.prefix, path)),
+	}
+	_, err := storage.s3.HeadObject(params)
+	if err != nil {
+		aerr, ok := err.(awserr.Error)
+		if ok && aerr.Code() == s3.ErrCodeNoSuchKey {
+			return false, nil
+		}
+
+		return false, err
+	}
+
+	return true, nil
+}
+
+// ReadLink returns the symbolic link pointed to by path.
+// This simply reads text file created with SymLink
+func (storage *PublishedStorage) ReadLink(path string) (string, error) {
+	params := &s3.HeadObjectInput{
+		Bucket: aws.String(storage.bucket),
+		Key:    aws.String(filepath.Join(storage.prefix, path)),
+	}
+	output, err := storage.s3.HeadObject(params)
+	if err != nil {
+		return "", err
+	}
+
+	return aws.StringValue(output.Metadata["SymLink"]), nil
+}

s3/public_test.go

@@ -160,6 +160,13 @@ func (s *PublishedStorageSuite) TestRemove(c *C) {
 	c.Check(err, IsNil)
 
 	s.AssertNoFile(c, "a/b")
+
+	s.PutFile(c, "lala/xyz", []byte("test"))
+
+	errp := s.prefixedStorage.Remove("xyz")
+	c.Check(errp, IsNil)
+
+	s.AssertNoFile(c, "lala/xyz")
 }
 
 func (s *PublishedStorageSuite) TestRemovePlusWorkaround(c *C) {
@@ -257,4 +264,46 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
+
+	// for prefixed storage:
+	// first link from pool
+	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	// 2nd link from pool, providing wrong path for source file
+	//
+	// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
+	s.prefixedStorage.pathCache = nil
+	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+}
+
+func (s *PublishedStorageSuite) TestSymLink(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.SymLink("a/b", "a/b.link")
+	c.Check(err, IsNil)
+
+	var link string
+	link, err = s.storage.ReadLink("a/b.link")
+	c.Check(err, IsNil)
+	c.Check(link, Equals, "a/b")
+
+	c.Skip("copy not available in s3test")
+}
+
+func (s *PublishedStorageSuite) TestFileExists(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	exists, err := s.storage.FileExists("a/b")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+
+	exists, _ = s.storage.FileExists("a/b.invalid")
+	// Comment out as there is an error in s3test implementation
+	// c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
 }

swift/public.go

@@ -264,3 +264,49 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
 
 	return nil
 }
+
+// SymLink creates a copy of src file and adds link information as meta data
+func (storage *PublishedStorage) SymLink(src string, dst string) error {
+	srcObjectName := filepath.Join(storage.prefix, src)
+	dstObjectName := filepath.Join(storage.prefix, dst)
+
+	headers := map[string]string{
+		"SymLink": srcObjectName,
+	}
+	_, err := storage.conn.ObjectCopy(storage.container, srcObjectName, storage.container, dstObjectName, headers)
+	if err != nil {
+		return fmt.Errorf("error symlinking %s -> %s in %s: %s", srcObjectName, dstObjectName, storage, err)
+	}
+	return err
+}
+
+// HardLink using symlink functionality as hard links do not exist
+func (storage *PublishedStorage) HardLink(src string, dst string) error {
+	return storage.SymLink(src, dst)
+}
+
+// FileExists returns true if path exists
+func (storage *PublishedStorage) FileExists(path string) (bool, error) {
+	_, _, err := storage.conn.Object(storage.container, filepath.Join(storage.prefix, path))
+
+	if err != nil {
+		if err == swift.ObjectNotFound {
+			return false, nil
+		}
+
+		return false, err
+	}
+
+	return true, nil
+}
+
+// ReadLink returns the symbolic link pointed to by path
+func (storage *PublishedStorage) ReadLink(path string) (string, error) {
+	srcObjectName := filepath.Join(storage.prefix, path)
+	_, headers, err := storage.conn.Object(storage.container, srcObjectName)
+	if err != nil {
+		return "", fmt.Errorf("error reading symlink %s in %s: %s", srcObjectName, storage, err)
+	}
+
+	return headers["SymLink"], nil
+}

swift/public_test.go

@@ -194,3 +194,39 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	c.Check(err, IsNil)
 	c.Check(data, DeepEquals, []byte("Spam"))
 }
+
+func (s *PublishedStorageSuite) TestSymLink(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to swift!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	err = s.storage.SymLink("a/b.txt", "a/b.txt.link")
+	c.Check(err, IsNil)
+
+	var link string
+	link, err = s.storage.ReadLink("a/b.txt.link")
+	c.Check(err, IsNil)
+	c.Check(link, Equals, "a/b")
+
+	c.Skip("copy not availbale in s3test")
+}
+
+func (s *PublishedStorageSuite) TestFileExists(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to swift!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	exists, err := s.storage.FileExists("a/b.txt")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+
+	exists, err = s.storage.FileExists("a/b.invalid")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}

system/files/nvidia.key

@@ -0,0 +1,52 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFnNWDEBEACiX68rxIWvqH3h2GykO25oK9BAqV8fDtb6lXEbw3eKx4g87BRz
+M3DQBA0S0IfkQ72ovJ33H50+gVTXuu+Zme5muWk72m3pApccZVDLqdzYlpWPruNb
+MC+IlWr70yo8Jw8Zr1ihbWjFvMbDJTkgqPt2djNq3xxvdiKoZlgnpLRKIpSu9iBQ
+lNoZLHxTQKFH4219L77prRogv2QV1ckBL5lDVOERJuHo4jHE8mm9/NZ6v3m2HGuu
+AEZ7T9nWlPGiAIP8Pww4ZRTJcBANcI2EFKPLdfP61HTH6w0kVMkoAaGlemadTDl3
+ZcLpUpTFLc+ko/2uQ1qVPx9QYyoMrorS3kUmlXrhsA7FvcB09aIcb+JX6SVkcbO5
+A5+baCa3owwUtFBXMHM5hqpLv4P3/GsuW6283YwLZCf53dJY4lJZePqzPGsvs/wS
+vhnZrFvb61i/Aqm0hjhVh7h6VNxUiE8geMcjxy29LtzajoyS0EPVxes4xZu0VbS7
+8LQyCNHSpS7TFmtVUQmbXqDN7cpiyr9+yutr0lZOMc7NYQt0nP/3RtYkWEob6wXa
+rVImHas1OYzlZymdO1uAnqkediS61E2vSD1OEq37/375FB/Q3AYXuNkQzDjYoJJz
+9wsv7Xp0bdPzQ/daLdIFNQXo5MmVIirsWM07JvbZaJhDOiJxGn0MPf11/QARAQAB
+tEBOVklESUEgQ09SUE9SQVRJT04gKE9wZW4gU291cmNlIFByb2plY3RzKSA8Y3Vk
+YXRvb2xzQG52aWRpYS5jb20+iQI4BBMBCgAiBQJZzVgxAhsPBgsJCggHAwUVCgkI
+CwUWAgMBAAIeAQIXgAAKCRDdyuBE95bssAh6EACgUCww2sr8sOztEHKhvdCsonXu
+THYbel3YlWmVDPbh4dA31xoRXlvSJptJzPi/zlTc9fkVSFGbEZbFRR4JjnwYTMLD
+ElMh5YRMYAoPVYhWGKIO4earu32GhFuPjfr6h+0xNaQeDPIbr7bPe/AEhLSdJMzI
+OuAifr7UaC65A6YlxfeaSqyt0HthYujoQ12cWxP998C5jkc0IN2tyLs/OD7HLHht
++lafqDSylykx63cw7jvsV/15rqZwVwjhkcxZyrKET32MTjXF3cxn7+TGpKS8B1k4
+a/EI7uXnncfSoma0dAT9bZM9JZbXQmSzCPDHHuVtnQ/3uh8VyenpigTFnrb20LCy
+6WzJd3O9lAZXLhvwF/By3a07WLzRtTZNaUpt37Anb0js2syr3lohbmK9i3xvuqZN
+zhGPbqu9IV+vFgSGyTHRJUSBlHKDGiCdOOHc20MLPW1yRCXbx0F4eS9TWchYyJkJ
+NNczD5DnEl/gsvL4NCRxa+oUyUhhJ1HpJ6YNmTsy6nAAKIC+6248o164GiavaR3z
+03RfaQayGHAUrBKi+PJBY7efgsZeYT8f+hyYrIC04MO8poBKS/GvSUL2QtVtj59N
+q+95gIptW2mZM8KRpt2huLH+QQ8SKr1vAECbpKJOwseqKmVyxX02iaSE8ifLE+tX
+FE8YgS3CZjWwy5PD0LkBDQRZzVxZAQgA5eJ4zclMsxMF/CRloniHKxyqyyMDlRSR
+088+WVtvcQ94ypAOh+vZ+GuTGOD6qQ23a3SrBkdi1gfilWQaQu3KMe8DCpCLavB3
+ipLKIa9ddFFFojdN43mg6JJwi1crVu/08uQfZwP7abo9/C4L00SH8VoF1Tfwv5sQ
+/LbcMT2kPgHvur9B11LiU1ZL7EF0Bam+Yt5VEr6vHK/psUkR3VOlpKvDzc36V6uk
+jm0qTODqoIaYhDOhUwf4PDWMj9R81VtTFGGLRXLDkWVEAOGihuar4kmhkDuNYG6/
+auyXLWx2jcQpMUSCUXuYAhbsjmyG7wFlHAnpvq5UhnNI23eT0APXwwARAQABiQNE
+BBgBCgAPBQJZzVxZAhsCBQkB4TOAASkJEN3K4ET3luywwF0gBBkBCgAGBQJZzVxZ
+AAoJEMRbFnagTqVSnGgIAJLCAMIwwK70tFoSc0eF1bMeb3tXQohkSVztcMU8mzEM
+qejGnU+4tva5H5XOvoO2oVsuY3ycn7TjQ8wa/9G/ZxKuB6zDxRZBOXPs0eCnDuSx
+rOPknlA3pFCMjihss92IjDChtEkXTSWactdAu+R1kIYbDyV9JVW2i6B/4L6f6tMc
+0XdxN2kcQqqSRjQhDBefHiU8mnFXAWDgeu/BToyBzlBJSVCZeAEXHxODdIoknNeH
+E5lhGTcFQz/p946BKSav991rq2Fl02RaBHFu3LL/BJKbJCz8DoMmVtY1eWYLsKO1
+mf9FdBdl36i9RcfxHH1zVQHrBWVR/guDkvARoOGByrv0OhAAhlwwrZC5Ztotk0Yi
+uu3V4AZDKLZord7AgoJoXsHc1C6uCJ+WDNEHsD94fTDeMs0n/xg3oowRJ2O7PVXA
+lW7asIimKtHKK8+X+G9z15R2ab3AOfsFILd3GYhVc2lKgOIR1gMg24LnLQ/RVKne
+WfP04HL0sYPaCoEMll3HPKEDqIjFKROSJCsyfWM8UsX+BiDVNc/qjCe1syShtISR
+nUNa687MG9hJMiGr4iEaey0nEysUYt/NA7hc1iyKyq+A0zPVDBuVfEWXiMHJgwjP
+DAIRIa88No1BcYtR1B1weS6m4uggbyi4aJSxnxNg/EoqJv3NEyXfOLz+bGF65B/A
+/l4gWEXzu9Vb+027+ZFTymEa8dkdqYclfnqdYuhYbOLsqVQM+ETVTPhZVos28nM3
+ivAE7EqreB+wj/V8ObN4vqzhaU0zl4ElgdyN0GYPR2mbIr4dgAgfQP66yoQvDTxK
+DPFzSNC7x4ehDnowy4CrK9LKBdHISR8vorSL10MD4ehdFR3308hyUgTo8/0IUkxa
+X5k2Oq99CGhIbuCFttnr1DCKNremMJVzkado7hx/C3Z1liaExG2dvnMrZBiN/S6V
+Ct7CJcRi+BjUy+um8P7hJNkN7uqvfax9MgF/pvdvCGmygsQAxUoenbAY/WP1Xz44
+Os8DlMDE6fGy/6l+pHuHpT52NxE=
+=cDVy
+-----END PGP PUBLIC KEY BLOCK-----

system/lib.py

@@ -82,6 +82,12 @@ class BaseTest(object):
 
     fixtureDBDir = os.path.join(os.environ["HOME"], "aptly-fixture-db")
     fixturePoolDir = os.path.join(os.environ["HOME"], "aptly-fixture-pool")
+    fixtureGpgKeys = ["debian-archive-keyring.gpg",
+                      "launchpad.key",
+                      "flat.key",
+                      "pagerduty.key",
+                      "nvidia.key",
+                      "jenkins.key"]
 
     outputMatchPrepare = None
 
@@ -132,12 +138,8 @@ class BaseTest(object):
                                                      self.fixtureWebServer))
 
         if self.fixtureGpg:
-            self.run_cmd(["gpg", "--no-default-keyring", "--trust-model", "always", "--batch", "--keyring", "aptlytest.gpg", "--import",
-                          os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "debian-archive-keyring.gpg"),
-                          os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "launchpad.key"),
-                          os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "flat.key"),
-                          os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "pagerduty.key"),
-                          os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "jenkins.key")])
+            self.run_cmd(["gpg", "--no-default-keyring", "--trust-model", "always", "--batch", "--keyring", "aptlytest.gpg", "--import"] +
+                         [os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", key) for key in self.fixtureGpgKeys])
 
         if hasattr(self, "fixtureCmds"):
             for cmd in self.fixtureCmds:
@@ -194,7 +196,7 @@ class BaseTest(object):
     def check_output(self):
         try:
             self.verify_match(self.get_gold(), self.output, match_prepare=self.outputMatchPrepare)
-        except:
+        except:  # noqa: E722
             if self.captureResults:
                 if self.outputMatchPrepare is not None:
                     self.output = self.outputMatchPrepare(self.output)
@@ -207,7 +209,7 @@ class BaseTest(object):
         output = self.run_cmd(command, expected_code=expected_code)
         try:
             self.verify_match(self.get_gold(gold_name), output, match_prepare)
-        except:
+        except:  # noqa: E722
             if self.captureResults:
                 if match_prepare is not None:
                     output = match_prepare(output)
@@ -228,7 +230,7 @@ class BaseTest(object):
         try:
 
             self.verify_match(self.get_gold(gold_name), contents, match_prepare=match_prepare)
-        except:
+        except:  # noqa: E722
             if self.captureResults:
                 if match_prepare is not None:
                     contents = match_prepare(contents)
@@ -241,7 +243,7 @@ class BaseTest(object):
         contents = open(self.checkedFile, "r").read()
         try:
             self.verify_match(self.get_gold(), contents)
-        except:
+        except:  # noqa: E722
             if self.captureResults:
                 with open(self.get_gold_filename(), "w") as f:
                     f.write(contents)

system/run.py

@@ -1,4 +1,4 @@
-#!/usr/local/bin/python
+#!/usr/bin/env python
 
 import glob
 import importlib
@@ -99,7 +99,8 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
         print "\nFAILURES (%d):" % (len(fails), )
 
         for (test, t, typ, val, tb, testModule) in fails:
-            print "%s:%s %s" % (test, t.__class__.__name__, testModule.__name__ + ": " + t.__doc__.strip())
+            doc = t.__doc__ or ''
+            print "%s:%s %s" % (test, t.__class__.__name__, testModule.__name__ + ": " + doc.strip())
             traceback.print_exception(typ, val, tb)
             print "=" * 60
 

system/s3_lib.py

@@ -18,6 +18,8 @@ class S3Test(BaseTest):
     BaseTest + support for S3
     """
 
+    s3Overrides = {}
+
     def fixture_available(self):
         return super(S3Test, self).fixture_available() and s3_conn is not None
 
@@ -31,6 +33,8 @@ class S3Test(BaseTest):
             }
         }}
 
+        self.configOverride["S3PublishEndpoints"]["test1"].update(**self.s3Overrides)
+
         super(S3Test, self).prepare()
 
     def shutdown(self):

system/t03_help/MainHelpTest_gold

@@ -14,10 +14,10 @@ Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
   -db-open-attempts=10: number of attempts to open DB if it's locked by other instance
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
-  -dep-follow-recommends=false: when processing dependencies, follow Recommends
-  -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
-  -dep-follow-suggests=false: when processing dependencies, follow Suggests
-  -dep-verbose-resolve=false: when processing dependencies, print detailed logs
+  -dep-follow-all-variants: when processing dependencies, follow a & b if dependency is 'a|b'
+  -dep-follow-recommends: when processing dependencies, follow Recommends
+  -dep-follow-source: when processing dependencies, follow from binary to Source packages
+  -dep-follow-suggests: when processing dependencies, follow Suggests
+  -dep-verbose-resolve: when processing dependencies, print detailed logs
   -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
 

system/t03_help/MainTest_gold

@@ -22,10 +22,10 @@ Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
   -db-open-attempts=10: number of attempts to open DB if it's locked by other instance
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
-  -dep-follow-recommends=false: when processing dependencies, follow Recommends
-  -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
-  -dep-follow-suggests=false: when processing dependencies, follow Suggests
-  -dep-verbose-resolve=false: when processing dependencies, print detailed logs
+  -dep-follow-all-variants: when processing dependencies, follow a & b if dependency is 'a|b'
+  -dep-follow-recommends: when processing dependencies, follow Recommends
+  -dep-follow-source: when processing dependencies, follow from binary to Source packages
+  -dep-follow-suggests: when processing dependencies, follow Suggests
+  -dep-verbose-resolve: when processing dependencies, print detailed logs
   -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
 ERROR: unable to parse command

system/t03_help/MirrorCreateHelpTest_gold

@@ -16,18 +16,18 @@ Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
   -db-open-attempts=10: number of attempts to open DB if it's locked by other instance
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
-  -dep-follow-recommends=false: when processing dependencies, follow Recommends
-  -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
-  -dep-follow-suggests=false: when processing dependencies, follow Suggests
-  -dep-verbose-resolve=false: when processing dependencies, print detailed logs
+  -dep-follow-all-variants: when processing dependencies, follow a & b if dependency is 'a|b'
+  -dep-follow-recommends: when processing dependencies, follow Recommends
+  -dep-follow-source: when processing dependencies, follow from binary to Source packages
+  -dep-follow-suggests: when processing dependencies, follow Suggests
+  -dep-verbose-resolve: when processing dependencies, print detailed logs
   -filter="": filter packages in mirror
-  -filter-with-deps=false: when filtering, include dependencies of matching packages as well
-  -force-architectures=false: (only with architecture list) skip check that requested architectures are listed in Release file
-  -force-components=false: (only with component list) skip check that requested components are listed in Release file
+  -filter-with-deps: when filtering, include dependencies of matching packages as well
+  -force-architectures: (only with architecture list) skip check that requested architectures are listed in Release file
+  -force-components: (only with component list) skip check that requested components are listed in Release file
   -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
-  -ignore-signatures=false: disable verification of Release file signatures
+  -ignore-signatures: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
-  -with-sources=false: download source packages in addition to binary packages
-  -with-udebs=false: download .udeb packages (Debian installer support)
+  -with-sources: download source packages in addition to binary packages
+  -with-udebs: download .udeb packages (Debian installer support)
 

system/t03_help/MirrorCreateTest_gold

@@ -7,18 +7,18 @@ Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
   -db-open-attempts=10: number of attempts to open DB if it's locked by other instance
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
-  -dep-follow-recommends=false: when processing dependencies, follow Recommends
-  -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
-  -dep-follow-suggests=false: when processing dependencies, follow Suggests
-  -dep-verbose-resolve=false: when processing dependencies, print detailed logs
+  -dep-follow-all-variants: when processing dependencies, follow a & b if dependency is 'a|b'
+  -dep-follow-recommends: when processing dependencies, follow Recommends
+  -dep-follow-source: when processing dependencies, follow from binary to Source packages
+  -dep-follow-suggests: when processing dependencies, follow Suggests
+  -dep-verbose-resolve: when processing dependencies, print detailed logs
   -filter="": filter packages in mirror
-  -filter-with-deps=false: when filtering, include dependencies of matching packages as well
-  -force-architectures=false: (only with architecture list) skip check that requested architectures are listed in Release file
-  -force-components=false: (only with component list) skip check that requested components are listed in Release file
+  -filter-with-deps: when filtering, include dependencies of matching packages as well
+  -force-architectures: (only with architecture list) skip check that requested architectures are listed in Release file
+  -force-components: (only with component list) skip check that requested components are listed in Release file
   -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
-  -ignore-signatures=false: disable verification of Release file signatures
+  -ignore-signatures: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
-  -with-sources=false: download source packages in addition to binary packages
-  -with-udebs=false: download .udeb packages (Debian installer support)
+  -with-sources: download source packages in addition to binary packages
+  -with-udebs: download .udeb packages (Debian installer support)
 ERROR: unable to parse command

system/t03_help/MirrorHelpTest_gold

@@ -18,9 +18,9 @@ Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
   -db-open-attempts=10: number of attempts to open DB if it's locked by other instance
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
-  -dep-follow-recommends=false: when processing dependencies, follow Recommends
-  -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
-  -dep-follow-suggests=false: when processing dependencies, follow Suggests
-  -dep-verbose-resolve=false: when processing dependencies, print detailed logs
+  -dep-follow-all-variants: when processing dependencies, follow a & b if dependency is 'a|b'
+  -dep-follow-recommends: when processing dependencies, follow Recommends
+  -dep-follow-source: when processing dependencies, follow from binary to Source packages
+  -dep-follow-suggests: when processing dependencies, follow Suggests
+  -dep-verbose-resolve: when processing dependencies, print detailed logs
   -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)

system/t03_help/MirrorTest_gold

@@ -18,10 +18,10 @@ Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
   -db-open-attempts=10: number of attempts to open DB if it's locked by other instance
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
-  -dep-follow-recommends=false: when processing dependencies, follow Recommends
-  -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
-  -dep-follow-suggests=false: when processing dependencies, follow Suggests
-  -dep-verbose-resolve=false: when processing dependencies, print detailed logs
+  -dep-follow-all-variants: when processing dependencies, follow a & b if dependency is 'a|b'
+  -dep-follow-recommends: when processing dependencies, follow Recommends
+  -dep-follow-source: when processing dependencies, follow from binary to Source packages
+  -dep-follow-suggests: when processing dependencies, follow Suggests
+  -dep-verbose-resolve: when processing dependencies, print detailed logs
   -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
 ERROR: unable to parse command

system/t03_help/WrongFlagTest_gold

@@ -8,18 +8,18 @@ Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
   -db-open-attempts=10: number of attempts to open DB if it's locked by other instance
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
-  -dep-follow-recommends=false: when processing dependencies, follow Recommends
-  -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
-  -dep-follow-suggests=false: when processing dependencies, follow Suggests
-  -dep-verbose-resolve=false: when processing dependencies, print detailed logs
+  -dep-follow-all-variants: when processing dependencies, follow a & b if dependency is 'a|b'
+  -dep-follow-recommends: when processing dependencies, follow Recommends
+  -dep-follow-source: when processing dependencies, follow from binary to Source packages
+  -dep-follow-suggests: when processing dependencies, follow Suggests
+  -dep-verbose-resolve: when processing dependencies, print detailed logs
   -filter="": filter packages in mirror
-  -filter-with-deps=false: when filtering, include dependencies of matching packages as well
-  -force-architectures=false: (only with architecture list) skip check that requested architectures are listed in Release file
-  -force-components=false: (only with component list) skip check that requested components are listed in Release file
+  -filter-with-deps: when filtering, include dependencies of matching packages as well
+  -force-architectures: (only with architecture list) skip check that requested architectures are listed in Release file
+  -force-components: (only with component list) skip check that requested components are listed in Release file
   -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
-  -ignore-signatures=false: disable verification of Release file signatures
+  -ignore-signatures: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
-  -with-sources=false: download source packages in addition to binary packages
-  -with-udebs=false: download .udeb packages (Debian installer support)
+  -with-sources: download source packages in addition to binary packages
+  -with-udebs: download .udeb packages (Debian installer support)
 ERROR: unable to parse flags

system/t04_mirror/EditMirror10Test_gold

@@ -0,0 +1,2 @@
+Downloading ftp://ftp.ch.debian.org/debian/dists/wheezy/Release...
+Mirror [mirror10]: ftp://ftp.ch.debian.org/debian/ wheezy successfully updated.

system/t04_mirror/SearchMirror4Test_gold

@@ -85,6 +85,7 @@ tar_1.26+dfsg-0.1_amd64
 tar_1.26+dfsg-0.1_i386
 ttf-bitstream-vera_1.10-8_all
 ttf-dejavu-core_2.33-3_all
+ttf-freefont_20120503-1_all
 ucf_3.0025+nmu3_all
 x11-common_1:7.7+3~deb7u1_all
 xfonts-encodings_1:1.0.4-1_all

system/t04_mirror/UpdateMirror13Test_gold

@@ -3,59 +3,59 @@
 Building download queue...
 Download queue: 52 items (19.79 MiB)
 Downloading & parsing package files...
-Downloading http://repo.varnish-cache.org/debian/dists/wheezy/Release...
-Downloading http://repo.varnish-cache.org/debian/dists/wheezy/varnish-3.0/binary-amd64/Packages.bz2...
-Downloading http://repo.varnish-cache.org/debian/dists/wheezy/varnish-3.0/binary-i386/Packages.bz2...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_1.16.0~wheezy_all.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_2.2.0~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_2.2.0~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_2.2.1+nmu1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_2.2.1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_2.2.1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_3.0.0~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_3.0.1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.3-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.3-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.4-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.4-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.5-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.5-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.6-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.6-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.7-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.7-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.3-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.3-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.4-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.4-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.5-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.5-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.6-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.6-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.7-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.7-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.3-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.3-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.4-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.4-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.5-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.5-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.6-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.6-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.7-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.7-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-doc_3.0.4-1~wheezy_all.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-doc_3.0.5-1~wheezy_all.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-doc_3.0.6-1~wheezy_all.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-doc_3.0.7-1~wheezy_all.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.3-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.3-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.4-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.4-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.5-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.5-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.6-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.6-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.7-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.7-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/Release...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-amd64/Packages.bz2...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-i386/Packages.bz2...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.4-1%29/libvarnishapi-dev_3.0.4-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.4-1%29/libvarnishapi-dev_3.0.4-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.4-1%29/libvarnishapi1_3.0.4-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.4-1%29/libvarnishapi1_3.0.4-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.5-1%29/libvarnishapi-dev_3.0.5-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.5-1%29/libvarnishapi-dev_3.0.5-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.5-1%29/libvarnishapi1_3.0.5-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.5-1%29/libvarnishapi1_3.0.5-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi-dev_3.0.6-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi-dev_3.0.6-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi1_3.0.6-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi1_3.0.6-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi-dev_3.0.7-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi-dev_3.0.7-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi1_3.0.7-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi1_3.0.7-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish/libvarnishapi-dev_3.0.3-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish/libvarnishapi-dev_3.0.3-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish/libvarnishapi1_3.0.3-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish/libvarnishapi1_3.0.3-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.4-1%29/varnish-dbg_3.0.4-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.4-1%29/varnish-dbg_3.0.4-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.4-1%29/varnish-doc_3.0.4-1~wheezy_all.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.4-1%29/varnish_3.0.4-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.4-1%29/varnish_3.0.4-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.5-1%29/varnish-dbg_3.0.5-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.5-1%29/varnish-dbg_3.0.5-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.5-1%29/varnish-doc_3.0.5-1~wheezy_all.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.5-1%29/varnish_3.0.5-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.5-1%29/varnish_3.0.5-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish-dbg_3.0.6-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish-dbg_3.0.6-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish-doc_3.0.6-1~wheezy_all.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish_3.0.6-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish_3.0.6-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-dbg_3.0.7-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-dbg_3.0.7-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-doc_3.0.7-1~wheezy_all.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish_3.0.7-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish_3.0.7-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%281.16.0%29/varnish-agent_1.16.0~wheezy_all.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.0%29/varnish-agent_2.2.0~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.0%29/varnish-agent_2.2.0~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.1%29/varnish-agent_2.2.1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.1%29/varnish-agent_2.2.1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.1+nmu1%29/varnish-agent_2.2.1+nmu1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%283.0.0%29/varnish-agent_3.0.0~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%283.0.1%29/varnish-agent_3.0.1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish-dbg_3.0.3-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish-dbg_3.0.3-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish_3.0.3-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish_3.0.3-1~wheezy_i386.deb...
 Mirror `varnish` has been successfully updated.

system/t04_mirror/UpdateMirror14Test_gold

@@ -3,7 +3,7 @@
 Building download queue...
 Download queue: 0 items (0 B)
 Downloading & parsing package files...
-Downloading http://repo.varnish-cache.org/debian/dists/wheezy/Release...
-Downloading http://repo.varnish-cache.org/debian/dists/wheezy/varnish-3.0/binary-amd64/Packages.bz2...
-Downloading http://repo.varnish-cache.org/debian/dists/wheezy/varnish-3.0/binary-i386/Packages.bz2...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/Release...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-amd64/Packages.bz2...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-i386/Packages.bz2...
 Mirror `varnish` has been successfully updated.

system/t04_mirror/UpdateMirror19Test_gold

@@ -1,7 +1,7 @@
 Downloading http://packages.pagerduty.com/pdagent/deb/InRelease...
 Downloading http://packages.pagerduty.com/pdagent/deb/Release...
 Downloading http://packages.pagerduty.com/pdagent/deb/Release.gpg...
-gpgv:  RSA key ID 76203C00
+gpgv:  RSA key ID F8253540
 gpgv: Good signature from "Package Maintainer (PagerDuty, Inc.) <packages@pagerduty.com>"
 Downloading & parsing package files...
 Downloading http://packages.pagerduty.com/pdagent/deb/Packages.gz...

system/t04_mirror/UpdateMirror1Test_gold

@@ -3,59 +3,59 @@
 Building download queue...
 Download queue: 52 items (19.79 MiB)
 Downloading & parsing package files...
-Downloading http://repo.varnish-cache.org/debian/dists/wheezy/Release...
-Downloading http://repo.varnish-cache.org/debian/dists/wheezy/varnish-3.0/binary-amd64/Packages.bz2...
-Downloading http://repo.varnish-cache.org/debian/dists/wheezy/varnish-3.0/binary-i386/Packages.bz2...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_1.16.0~wheezy_all.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_2.2.0~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_2.2.0~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_2.2.1+nmu1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_2.2.1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_2.2.1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_3.0.0~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish-agent/varnish-agent_3.0.1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.3-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.3-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.4-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.4-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.5-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.5-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.6-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.6-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.7-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi-dev_3.0.7-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.3-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.3-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.4-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.4-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.5-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.5-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.6-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.6-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.7-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/libvarnishapi1_3.0.7-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.3-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.3-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.4-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.4-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.5-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.5-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.6-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.6-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.7-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-dbg_3.0.7-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-doc_3.0.4-1~wheezy_all.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-doc_3.0.5-1~wheezy_all.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-doc_3.0.6-1~wheezy_all.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish-doc_3.0.7-1~wheezy_all.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.3-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.3-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.4-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.4-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.5-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.5-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.6-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.6-1~wheezy_i386.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.7-1~wheezy_amd64.deb...
-Downloading http://repo.varnish-cache.org/debian/pool/varnish-3.0/v/varnish/varnish_3.0.7-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/Release...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-amd64/Packages.bz2...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-i386/Packages.bz2...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.4-1%29/libvarnishapi-dev_3.0.4-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.4-1%29/libvarnishapi-dev_3.0.4-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.4-1%29/libvarnishapi1_3.0.4-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.4-1%29/libvarnishapi1_3.0.4-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.5-1%29/libvarnishapi-dev_3.0.5-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.5-1%29/libvarnishapi-dev_3.0.5-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.5-1%29/libvarnishapi1_3.0.5-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.5-1%29/libvarnishapi1_3.0.5-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi-dev_3.0.6-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi-dev_3.0.6-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi1_3.0.6-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.6-1%29/libvarnishapi1_3.0.6-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi-dev_3.0.7-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi-dev_3.0.7-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi1_3.0.7-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish%20%283.0.7-1%29/libvarnishapi1_3.0.7-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish/libvarnishapi-dev_3.0.3-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish/libvarnishapi-dev_3.0.3-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish/libvarnishapi1_3.0.3-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/libv/varnish/libvarnishapi1_3.0.3-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.4-1%29/varnish-dbg_3.0.4-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.4-1%29/varnish-dbg_3.0.4-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.4-1%29/varnish-doc_3.0.4-1~wheezy_all.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.4-1%29/varnish_3.0.4-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.4-1%29/varnish_3.0.4-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.5-1%29/varnish-dbg_3.0.5-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.5-1%29/varnish-dbg_3.0.5-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.5-1%29/varnish-doc_3.0.5-1~wheezy_all.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.5-1%29/varnish_3.0.5-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.5-1%29/varnish_3.0.5-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish-dbg_3.0.6-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish-dbg_3.0.6-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish-doc_3.0.6-1~wheezy_all.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish_3.0.6-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.6-1%29/varnish_3.0.6-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-dbg_3.0.7-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-dbg_3.0.7-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish-doc_3.0.7-1~wheezy_all.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish_3.0.7-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish%20%283.0.7-1%29/varnish_3.0.7-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%281.16.0%29/varnish-agent_1.16.0~wheezy_all.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.0%29/varnish-agent_2.2.0~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.0%29/varnish-agent_2.2.0~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.1%29/varnish-agent_2.2.1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.1%29/varnish-agent_2.2.1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%282.2.1+nmu1%29/varnish-agent_2.2.1+nmu1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%283.0.0%29/varnish-agent_3.0.0~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish-agent%20%283.0.1%29/varnish-agent_3.0.1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish-dbg_3.0.3-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish-dbg_3.0.3-1~wheezy_i386.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish_3.0.3-1~wheezy_amd64.deb...
+Downloading https://packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish_3.0.3-1~wheezy_i386.deb...
 Mirror `varnish` has been successfully updated.

system/t04_mirror/UpdateMirror21Test_gold

@@ -1,7 +1,7 @@
 Downloading http://packages.pagerduty.com/pdagent/deb/InRelease...
 Downloading http://packages.pagerduty.com/pdagent/deb/Release...
 Downloading http://packages.pagerduty.com/pdagent/deb/Release.gpg...
-openpgp:  RSA key ID 872A32ED76203C00
+openpgp:  RSA key ID AE0396CFF8253540
 openpgp: Good signature from "Package Maintainer (PagerDuty, Inc.) <packages@pagerduty.com>"
 Downloading & parsing package files...
 Downloading http://packages.pagerduty.com/pdagent/deb/Packages.gz...

system/t04_mirror/UpdateMirror22Test_gold

@@ -0,0 +1,11 @@
+Downloading https://nvidia.github.io/libnvidia-container/ubuntu16.04/amd64/InRelease...
+openpgp:  RSA key ID C45B1676A04EA552
+openpgp: Good signature from "NVIDIA CORPORATION (Open Source Projects) <cudatools@nvidia.com>"
+Downloading & parsing package files...
+Downloading https://nvidia.github.io/libnvidia-container/ubuntu16.04/amd64/Packages.xz...
+Applying filter...
+Packages filtered: 8 -> 0.
+Building download queue...
+Download queue: 0 items (0 B)
+
+Mirror `libnvidia-container` has been successfully updated.

system/t04_mirror/edit.py

@@ -65,7 +65,7 @@ class EditMirror6Test(BaseTest):
     edit mirror: change architectures
     """
     fixtureDB = True
-    runCmd = "aptly mirror edit -architectures=amd64,s390 wheezy-main"
+    runCmd = "aptly mirror edit -ignore-signatures -architectures=amd64,s390 wheezy-main"
 
     def check(self):
         self.check_output()
@@ -77,7 +77,7 @@ class EditMirror7Test(BaseTest):
     edit mirror: change architectures to missing archs
     """
     fixtureDB = True
-    runCmd = "aptly mirror edit -architectures=amd64,x56 wheezy-main"
+    runCmd = "aptly mirror edit -ignore-signatures -architectures=amd64,x56 wheezy-main"
     expectedCode = 1
 
 
@@ -101,3 +101,11 @@ class EditMirror9Test(BaseTest):
     fixtureGpg = True
     runCmd = "aptly mirror edit -with-udebs mirror9"
     expectedCode = 1
+
+
+class EditMirror10Test(BaseTest):
+    """
+    edit mirror: change archive url
+    """
+    fixtureCmds = ["aptly mirror create -ignore-signatures mirror10 ftp://ftp.ru.debian.org/debian wheezy main"]
+    runCmd = "aptly mirror edit -ignore-signatures -archive-url ftp://ftp.ch.debian.org/debian mirror10"

system/t04_mirror/update.py

@@ -16,7 +16,7 @@ class UpdateMirror1Test(BaseTest):
     """
     longTest = False
     fixtureCmds = [
-        "aptly -architectures=i386,amd64 mirror create --ignore-signatures varnish http://repo.varnish-cache.org/debian/ wheezy varnish-3.0",
+        "aptly -architectures=i386,amd64 mirror create --ignore-signatures varnish https://packagecloud.io/varnishcache/varnish30/debian/ wheezy main",
     ]
     runCmd = "aptly mirror update --ignore-signatures varnish"
 
@@ -187,7 +187,7 @@ class UpdateMirror13Test(BaseTest):
     """
     longTest = False
     fixtureCmds = [
-        "aptly -architectures=i386,amd64 mirror create --ignore-signatures varnish http://repo.varnish-cache.org/debian/ wheezy varnish-3.0",
+        "aptly -architectures=i386,amd64 mirror create --ignore-signatures varnish https://packagecloud.io/varnishcache/varnish30/debian/ wheezy main",
     ]
     runCmd = "aptly mirror update --ignore-signatures --skip-existing-packages varnish"
 
@@ -201,7 +201,7 @@ class UpdateMirror14Test(BaseTest):
     """
     longTest = False
     fixtureCmds = [
-        "aptly -architectures=i386,amd64 mirror create --ignore-signatures varnish http://repo.varnish-cache.org/debian/ wheezy varnish-3.0",
+        "aptly -architectures=i386,amd64 mirror create --ignore-signatures varnish https://packagecloud.io/varnishcache/varnish30/debian/ wheezy main",
         "aptly mirror update --ignore-signatures --skip-existing-packages varnish"
     ]
     runCmd = "aptly mirror update --ignore-signatures --skip-existing-packages varnish"
@@ -360,3 +360,16 @@ class UpdateMirror21Test(BaseTest):
 
     def output_processor(self, output):
         return "\n".join(line for line in output.split("\n") if ".deb" not in line)
+
+
+class UpdateMirror22Test(BaseTest):
+    """
+    update mirrors: SHA512 checksums only
+    """
+    configOverride = {"gpgProvider": "internal"}
+    fixtureGpg = True
+    fixtureCmds = [
+        "aptly mirror create --keyring=aptlytest.gpg --filter=nomatch libnvidia-container https://nvidia.github.io/libnvidia-container/ubuntu16.04/amd64 ./"
+    ]
+    runCmd = "aptly mirror update --keyring=aptlytest.gpg libnvidia-container"
+    outputMatchPrepare = filterOutSignature

system/t05_snapshot/PullSnapshot15Test_gold

@@ -2,6 +2,7 @@
 
     [snap1]: Snapshot from mirror [wheezy-main]: http://mirror.yandex.ru/debian/ wheezy
     [snap2]: Snapshot from mirror [wheezy-backports]: http://mirror.yandex.ru/debian/ wheezy-backports
+Already satisfied dependency: init-system-helpers (>= 1.18~) [i386] with [init-system-helpers_1.18~bpo70+1_all]
 Building indexes...
 Dependencies would be pulled into snapshot:
 Injecting package: init-system-helpers_1.18~bpo70+1_all

system/t05_snapshot/SearchSnapshot4Test_gold

@@ -85,6 +85,7 @@ tar_1.26+dfsg-0.1_amd64
 tar_1.26+dfsg-0.1_i386
 ttf-bitstream-vera_1.10-8_all
 ttf-dejavu-core_2.33-3_all
+ttf-freefont_20120503-1_all
 ucf_3.0025+nmu3_all
 x11-common_1:7.7+3~deb7u1_all
 xfonts-encodings_1:1.0.4-1_all

system/t06_publish/PublishDrop8Test_gold

@@ -0,0 +1,3 @@
+Removing ${HOME}/.aptly/public/dists/sq2...
+
+Published repository has been removed successfully.

system/t06_publish/PublishDrop9Test_gold

@@ -0,0 +1,4 @@
+Removing ${HOME}/.aptly/public/dists/sq1...
+Cleaning up prefix "." components main...
+
+Published repository has been removed successfully.

system/t06_publish/PublishSwitch14Test_binary

@@ -0,0 +1,29 @@
+
+ .
+ .
+ C-like language. Can perform smoothing, spline-fitting, or nonlinear fits,
+ Data files and self-defined functions can be manipulated by the internal
+ Gnuplot is a portable command-line driven interactive data and function
+ This package contains the terminal driver that enables gnuplot to plot
+ and can work with complex numbers.
+ for many printers, (La)TeX, (x)fig, Postscript, and so on. The X11-output
+ gnuplot.
+ images interactively under X11. Most users will want this, it is however
+ is packaged in gnuplot-x11.
+ packaged separately so that low-end systems don't need X installed to use
+ plotting utility that supports lots of output formats, including drivers
+Architecture: i386
+Depends: gnuplot-nox (>= 4.6.1-1~maverick2), libc6 (>= 2.11), libcairo2 (>= 1.6.0), libedit2 (>= 2.5.cvs.20010821-1), libgcc1 (>= 1:4.1.1), libgd2-noxpm (>= 2.0.36~rc1~dfsg) | libgd2-xpm (>= 2.0.36~rc1~dfsg), libglib2.0-0 (>= 2.12.0), liblua5.1-0, libpango1.0-0 (>= 1.14.0), libstdc++6 (>= 4.1.1), libwxbase2.8-0 (>= 2.8.11.0), libwxgtk2.8-0 (>= 2.8.11.0), libx11-6
+Description: Command-line driven interactive plotting program
+Filename: pool/main/g/gnuplot/gnuplot-x11_4.6.1-1~maverick2_i386.deb
+Installed-Size: 1604
+MD5sum: fcad938905d0ace50a6ce0c73b2c6583
+Maintainer: Debian Science Team <debian-science-maintainers@lists.alioth.debian.org>
+Package: gnuplot-x11
+Priority: optional
+Replaces: gnuplot (<< 4.0.0)
+SHA1: 02f9a93097a8f798a054e26154dbe5789088c069
+Section: math
+Size: 724388
+Source: gnuplot
+Version: 4.6.1-1~maverick2

system/t06_publish/PublishSwitch14Test_gold

@@ -0,0 +1,7 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+
+Publish for snapshot ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot.

system/t06_publish/PublishSwitch14Test_release

@@ -0,0 +1,11 @@
+Origin: LP-PPA-gladky-anton-gnuplot
+Label: . maverick
+Suite: maverick
+Codename: maverick
+Architectures: amd64 i386
+Components: main
+Description: Generated by aptly
+MD5Sum:
+SHA1:
+SHA256:
+SHA512:

system/t06_publish/PublishUpdate12Test_binary

@@ -0,0 +1,27 @@
+
+
+ (name, value) pairs from the user, via conventional methods such as
+ .
+ .
+ Boost version (currently 1.49).
+ Library to let program developers obtain program options, that is
+ This package forms part of the Boost C++ Libraries collection.
+ This package is a dependency package, which depends on Debian's default
+ command line and config file.
+Architecture: i386
+Depends: libboost-program-options1.49-dev
+Description: program options library for C++ (default version)
+Filename: pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb
+Homepage: http://www.boost.org/libs/program_options/
+Installed-Size: 26
+MD5sum: 0035d7822b2f8f0ec4013f270fd650c2
+Maintainer: Debian Boost Team <pkg-boost-devel@lists.alioth.debian.org>
+Package: libboost-program-options-dev
+Priority: optional
+SHA1: 36895eb64cfe89c33c0a2f7ac2f0c6e0e889e04b
+SHA256: c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12
+SHA512: d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c
+Section: libdevel
+Size: 2738
+Source: boost-defaults
+Version: 1.49.0.1

system/t06_publish/PublishUpdate12Test_gold

@@ -0,0 +1,7 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+
+Publish for local repo ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.

system/t06_publish/PublishUpdate12Test_release

@@ -0,0 +1,11 @@
+Origin: . maverick
+Label: . maverick
+Suite: maverick
+Codename: maverick
+Architectures: i386
+Components: main
+Description: Generated by aptly
+MD5Sum:
+SHA1:
+SHA256:
+SHA512:

system/t06_publish/S3Publish6Test_binary

@@ -0,0 +1,27 @@
+
+
+ (name, value) pairs from the user, via conventional methods such as
+ .
+ .
+ Boost version (currently 1.49).
+ Library to let program developers obtain program options, that is
+ This package forms part of the Boost C++ Libraries collection.
+ This package is a dependency package, which depends on Debian's default
+ command line and config file.
+Architecture: i386
+Depends: libboost-program-options1.49-dev
+Description: program options library for C++ (default version)
+Filename: pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb
+Homepage: http://www.boost.org/libs/program_options/
+Installed-Size: 26
+MD5sum: 0035d7822b2f8f0ec4013f270fd650c2
+Maintainer: Debian Boost Team <pkg-boost-devel@lists.alioth.debian.org>
+Package: libboost-program-options-dev
+Priority: optional
+SHA1: 36895eb64cfe89c33c0a2f7ac2f0c6e0e889e04b
+SHA256: c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12
+SHA512: d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c
+Section: libdevel
+Size: 2738
+Source: boost-defaults
+Version: 1.49.0.1

system/t06_publish/S3Publish6Test_gold

@@ -0,0 +1,8 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+Signing file 'Release' with gpg, please enter your passphrase when prompted:
+Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
+Cleaning up prefix "." components main...
+
+Publish for local repo s3:test1:./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.

system/t06_publish/S3Publish6Test_release

@@ -0,0 +1,11 @@
+Origin: . maverick
+Label: . maverick
+Suite: maverick
+Codename: maverick
+Architectures: i386
+Components: main
+Description: Generated by aptly
+MD5Sum:
+SHA1:
+SHA256:
+SHA512:

system/t06_publish/drop.py

@@ -143,3 +143,66 @@ class PublishDrop7Test(BaseTest):
         self.check_not_exists('public/ppa/smira/dists/')
         self.check_not_exists('public/ppa/smira/pool/')
         self.check_exists('public/ppa/smira/')
+
+
+class PublishDrop8Test(BaseTest):
+    """
+    publish drop: skip component cleanup
+    """
+    fixtureCmds = [
+        "aptly repo create local1",
+        "aptly repo create local2",
+        "aptly repo add local1 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb",
+        "aptly repo add local2 ${files}",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2",
+    ]
+    runCmd = "aptly publish drop -skip-cleanup sq2"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishDrop8Test, self).check()
+
+        self.check_exists('public/dists/sq1')
+        self.check_not_exists('public/dists/sq2')
+        self.check_exists('public/pool/main/')
+
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')
+
+
+class PublishDrop9Test(BaseTest):
+    """
+    publish drop: component cleanup after first cleanup skipped
+    """
+    fixtureCmds = [
+        "aptly repo create local1",
+        "aptly repo create local2",
+        "aptly repo create local3",
+        "aptly repo add local1 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb",
+        "aptly repo add local2 ${files}",
+        "aptly repo add local3 ${files}/libboost-program-options-dev_1.49.0.1_i386.deb",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq3 local3",
+        "aptly publish drop -skip-cleanup sq2"
+    ]
+    runCmd = "aptly publish drop sq1"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishDrop9Test, self).check()
+
+        self.check_not_exists('public/dists/sq1')
+        self.check_not_exists('public/dists/sq2')
+        self.check_exists('public/dists/sq3')
+        self.check_exists('public/pool/main/')
+
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')

system/t06_publish/s3.py

@@ -156,3 +156,42 @@ class S3Publish5Test(S3Test):
         self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
         self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
         self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')
+
+
+class S3Publish6Test(S3Test):
+    """
+    publish to S3: publish update removed some packages with SSE AES256
+    """
+    s3Overrides = {'encryptionMethod': 'AES256'}
+    fixtureCmds = [
+        "aptly repo create -distribution=maverick local-repo",
+        "aptly repo add local-repo ${files}/",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo s3:test1:",
+        "aptly repo remove local-repo pyspi"
+    ]
+    runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick s3:test1:"
+
+    def check(self):
+        super(S3Publish6Test, self).check()
+
+        self.check_exists('public/dists/maverick/InRelease')
+        self.check_exists('public/dists/maverick/Release')
+        self.check_exists('public/dists/maverick/Release.gpg')
+
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
+        self.check_exists('public/dists/maverick/main/source/Sources')
+        self.check_exists('public/dists/maverick/main/source/Sources.gz')
+        self.check_exists('public/dists/maverick/main/source/Sources.bz2')
+
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_not_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')
+
+        # verify contents except of sums
+        self.check_file_contents('public/dists/maverick/Release', 'release', match_prepare=strip_processor)
+        self.check_file_contents('public/dists/maverick/main/source/Sources', 'sources', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+        self.check_file_contents('public/dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))

system/t06_publish/snapshot.py

@@ -219,7 +219,7 @@ class PublishSnapshot5Test(BaseTest):
     fixtureCmds = [
         "aptly snapshot create snap5 from mirror gnuplot-maverick",
     ]
-    runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap5 ppa/smira"
+    runCmd = "aptly publish snapshot -acquire-by-hash -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap5 ppa/smira"
 
     gold_processor = BaseTest.expand_environ
 
@@ -231,12 +231,15 @@ class PublishSnapshot5Test(BaseTest):
         self.check_exists('public/ppa/smira/dists/squeeze/Release.gpg')
 
         self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/Packages')
+        self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/by-hash/MD5Sum/e98cd30fc76fbe7fa3ea25717efa1c92')
         self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/Packages.gz')
         self.check_exists('public/ppa/smira/dists/squeeze/main/binary-i386/Packages.bz2')
-        self.check_exists('public/ppa/smira/dists/squeeze/main/Contents-i386.gz')
         self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/Packages')
+        self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/by-hash/MD5Sum/ab073d1f73bed52e7356c91161e8667e')
         self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/Packages.gz')
         self.check_exists('public/ppa/smira/dists/squeeze/main/binary-amd64/Packages.bz2')
+
+        self.check_exists('public/ppa/smira/dists/squeeze/main/Contents-i386.gz')
         self.check_exists('public/ppa/smira/dists/squeeze/main/Contents-amd64.gz')
 
         self.check_exists('public/ppa/smira/pool/main/g/gnuplot/gnuplot-doc_4.6.1-1~maverick2_all.deb')

system/t06_publish/switch.py

@@ -451,3 +451,85 @@ class PublishSwitch13Test(BaseTest):
         self.check_not_exists('public/dists/maverick/main/Contents-i386.gz')
         self.check_exists('public/dists/maverick/main/binary-amd64/Packages')
         self.check_not_exists('public/dists/maverick/main/Contents-amd64.gz')
+
+
+class PublishSwitch14Test(BaseTest):
+    """
+    publish switch: removed some packages skipping cleanup
+    """
+    fixtureDB = True
+    fixturePool = True
+    fixtureCmds = [
+        "aptly snapshot create snap1 from mirror gnuplot-maverick",
+        "aptly snapshot create snap2 empty",
+        "aptly snapshot pull -no-deps -architectures=i386,amd64 snap2 snap1 snap3 gnuplot-x11",
+        "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap1",
+    ]
+    runCmd = "aptly publish switch -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-cleanup maverick snap3"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishSwitch14Test, self).check()
+
+        self.check_exists('public/dists/maverick/InRelease')
+        self.check_exists('public/dists/maverick/Release')
+        self.check_exists('public/dists/maverick/Release.gpg')
+
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
+        self.check_exists('public/dists/maverick/main/Contents-i386.gz')
+        self.check_exists('public/dists/maverick/main/binary-amd64/Packages')
+        self.check_exists('public/dists/maverick/main/binary-amd64/Packages.gz')
+        self.check_exists('public/dists/maverick/main/binary-amd64/Packages.bz2')
+        self.check_exists('public/dists/maverick/main/Contents-amd64.gz')
+
+        self.check_exists('public/pool/main/g/gnuplot/gnuplot-x11_4.6.1-1~maverick2_i386.deb')
+        self.check_exists('public/pool/main/g/gnuplot/gnuplot-x11_4.6.1-1~maverick2_amd64.deb')
+        self.check_exists('public/pool/main/g/gnuplot/gnuplot-nox_4.6.1-1~maverick2_i386.deb')
+        self.check_exists('public/pool/main/g/gnuplot/gnuplot-nox_4.6.1-1~maverick2_amd64.deb')
+
+        # verify contents except of sums
+        self.check_file_contents('public/dists/maverick/Release', 'release', match_prepare=strip_processor)
+        self.check_file_contents('public/dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+
+        # verify signatures
+        self.run_cmd(["gpg", "--no-auto-check-trustdb", "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"),
+                      "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/InRelease')])
+        self.run_cmd(["gpg", "--no-auto-check-trustdb",  "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"),
+                      "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release.gpg'),
+                      os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release')])
+
+        # verify sums
+        release = self.read_file('public/dists/maverick/Release').split("\n")
+        release = [l for l in release if l.startswith(" ")]
+        pathsSeen = set()
+        for l in release:
+            fileHash, fileSize, path = l.split()
+            pathsSeen.add(path)
+
+            fileSize = int(fileSize)
+
+            st = os.stat(os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/', path))
+            if fileSize != st.st_size:
+                raise Exception("file size doesn't match for %s: %d != %d" % (path, fileSize, st.st_size))
+
+            if len(fileHash) == 32:
+                h = hashlib.md5()
+            elif len(fileHash) == 40:
+                h = hashlib.sha1()
+            elif len(fileHash) == 64:
+                h = hashlib.sha256()
+            else:
+                h = hashlib.sha512()
+
+            h.update(self.read_file(os.path.join('public/dists/maverick', path)))
+
+            if h.hexdigest() != fileHash:
+                raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest()))
+
+        if pathsSeen != set(['main/binary-amd64/Packages', 'main/binary-i386/Packages', 'main/binary-i386/Packages.gz',
+                             'main/binary-amd64/Packages.gz', 'main/binary-amd64/Packages.bz2', 'main/binary-i386/Packages.bz2',
+                             'main/binary-amd64/Release', 'main/binary-i386/Release', 'main/Contents-amd64.gz',
+                             'main/Contents-i386.gz']):
+            raise Exception("path seen wrong: %r" % (pathsSeen, ))

system/t06_publish/update.py

@@ -95,7 +95,7 @@ class PublishUpdate2Test(BaseTest):
     fixtureCmds = [
         "aptly repo create local-repo",
         "aptly repo add local-repo ${files}/libboost-program-options-dev_1.49.0.1_i386.deb ${files}/pyspi_0.6.1-1.3.dsc",
-        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo",
+        "aptly publish repo -acquire-by-hash -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo",
         "aptly repo add local-repo ${files}/pyspi-0.6.1-1.3.stripped.dsc"
     ]
     runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick"
@@ -113,8 +113,14 @@ class PublishUpdate2Test(BaseTest):
         self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
         self.check_exists('public/dists/maverick/main/Contents-i386.gz')
         self.check_exists('public/dists/maverick/main/source/Sources')
+        self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources')
+        self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.old')
         self.check_exists('public/dists/maverick/main/source/Sources.gz')
+        self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.gz')
+        self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.gz.old')
         self.check_exists('public/dists/maverick/main/source/Sources.bz2')
+        self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.bz2')
+        self.check_exists('public/dists/maverick/main/source/by-hash/MD5Sum/Sources.bz2.old')
 
         self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
         self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
@@ -339,3 +345,83 @@ class PublishUpdate11Test(BaseTest):
 
         self.check_exists('public/dists/maverick/main/binary-i386/Packages')
         self.check_not_exists('public/dists/maverick/main/Contents-i386.gz')
+
+
+class PublishUpdate12Test(BaseTest):
+    """
+    publish update: removed some packages skipping cleanup
+    """
+    fixtureCmds = [
+        "aptly repo create local-repo",
+        "aptly repo add local-repo ${files}/",
+        "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick local-repo",
+        "aptly repo remove local-repo pyspi"
+    ]
+    runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-cleanup maverick"
+    gold_processor = BaseTest.expand_environ
+
+    def check(self):
+        super(PublishUpdate12Test, self).check()
+
+        self.check_exists('public/dists/maverick/InRelease')
+        self.check_exists('public/dists/maverick/Release')
+        self.check_exists('public/dists/maverick/Release.gpg')
+
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
+        self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
+        self.check_exists('public/dists/maverick/main/Contents-i386.gz')
+        self.check_exists('public/dists/maverick/main/source/Sources')
+        self.check_exists('public/dists/maverick/main/source/Sources.gz')
+        self.check_exists('public/dists/maverick/main/source/Sources.bz2')
+
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.dsc')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1-1.3.diff.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi_0.6.1.orig.tar.gz')
+        self.check_exists('public/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc')
+        self.check_exists('public/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb')
+
+        # verify contents except of sums
+        self.check_file_contents('public/dists/maverick/Release', 'release', match_prepare=strip_processor)
+        self.check_file_contents('public/dists/maverick/main/source/Sources', 'sources', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+        self.check_file_contents('public/dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
+
+        # verify signatures
+        self.run_cmd(["gpg", "--no-auto-check-trustdb", "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"),
+                      "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/InRelease')])
+        self.run_cmd(["gpg", "--no-auto-check-trustdb",  "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"),
+                      "--verify", os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release.gpg'),
+                      os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release')])
+
+        # verify sums
+        release = self.read_file('public/dists/maverick/Release').split("\n")
+        release = [l for l in release if l.startswith(" ")]
+        pathsSeen = set()
+        for l in release:
+            fileHash, fileSize, path = l.split()
+            pathsSeen.add(path)
+
+            fileSize = int(fileSize)
+
+            st = os.stat(os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/', path))
+            if fileSize != st.st_size:
+                raise Exception("file size doesn't match for %s: %d != %d" % (path, fileSize, st.st_size))
+
+            if len(fileHash) == 32:
+                h = hashlib.md5()
+            elif len(fileHash) == 40:
+                h = hashlib.sha1()
+            elif len(fileHash) == 64:
+                h = hashlib.sha256()
+            else:
+                h = hashlib.sha512()
+
+            h.update(self.read_file(os.path.join('public/dists/maverick', path)))
+
+            if h.hexdigest() != fileHash:
+                raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest()))
+
+        if pathsSeen != set(['main/binary-i386/Packages', 'main/binary-i386/Packages.bz2', 'main/binary-i386/Packages.gz',
+                             'main/source/Sources', 'main/source/Sources.gz', 'main/source/Sources.bz2',
+                             'main/binary-i386/Release', 'main/source/Release', 'main/Contents-i386.gz']):
+            raise Exception("path seen wrong: %r" % (pathsSeen, ))

system/t09_repo/AddRepo16Test_gold

@@ -0,0 +1,2 @@
+Loading packages...
+[+] pyspi_0.6.1-1.3_source added

system/t09_repo/AddRepo16Test_repo_show

@@ -0,0 +1,5 @@
+Name: repo2
+Comment: 
+Default Distribution: 
+Default Component: main
+Number of packages: 1