Fix one more system test on version.

Add a flag to unlock database after each API request

.travis.yml

@@ -1,31 +1,36 @@
+sudo: false
+
 language: go
 
 go:
-  - 1.3.3
+  - 1.4
+  - 1.5
   - tip
 
+addons:
+  apt:
+    packages:
+    - python-virtualenv
+    - graphviz
+
 env:
   global:
     - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
     - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
     - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
 before_install:
-  - sudo apt-get update -qq
-  - sudo apt-get install -y python-virtualenv graphviz
   - virtualenv env
   - . env/bin/activate
   - pip install boto requests python-swiftclient
 install:
   - make prepare
 
-
 script: make travis
 
 matrix:
     allow_failures:
         - go: tip
 
-
 notifications:
   webhooks:
     urls:

AUTHORS

@@ -15,3 +15,7 @@ List of contributors, in chronological order:
 * Michael Koval (https://github.com/mkoval)
 * Alexander Guy (https://github.com/alexanderguy)
 * Sebastien Badia (https://github.com/sbadia)
+* Szymon Sobik (https://github.com/sobczyk)
+* Paul Krohn (https://github.com/paul-krohn)
+* Vincent Bernat (https://github.com/vincentbernat)
+* x539 (https://github.com/x539)

Gomfile

@@ -1,21 +1,23 @@
-gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
-gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2'
-gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556'
-gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
 gom 'github.com/AlekSi/pointer', :commit => '5f6d527dae3d678b46fbb20331ddf44e2b841943'
+gom 'github.com/awalterschulze/gographviz', :commit => '20d1f693416d9be045340150094aa42035a41c9e'
 gom 'github.com/cheggaaa/pb', :commit => '2c1b74620cc58a81ac152ee2d322e28c806d81ed'
+gom 'github.com/DisposaBoy/JsonConfigReader', :commit => '33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4'
 gom 'github.com/gin-gonic/gin', :commit => 'b1758d3bfa09e61ddbc1c9a627e936eec6a170de'
 gom 'github.com/jlaffaye/ftp', :commit => 'fec71e62e457557fbe85cefc847a048d57815d76'
 gom 'github.com/julienschmidt/httprouter', :commit => '46807412fe50aaceb73bb57061c2230fd26a1640'
 gom 'github.com/mattn/go-shellwords', :commit => 'c7ca6f94add751566a61cf2199e1de78d4c3eee4'
-gom 'github.com/mitchellh/goamz/s3', :commit => 'e7664b32019f31fd1bdf33f9e85f28722f700405'
-gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
+gom 'github.com/mitchellh/goamz/s3', :commit => 'caaaea8b30ee15616494ee68abd5d8ebbbef05cf'
+gom 'github.com/mkrautz/goar', :commit => '282caa8bd9daba480b51f1d5a988714913b97aad'
+gom 'github.com/mxk/go-flowrate/flowrate', :commit => 'cca7078d478f8520f85629ad7c68962d31ed7682'
 gom 'github.com/ncw/swift', :commit => '384ef27c70645e285f8bb9d02276bf654d06027e'
+gom 'github.com/smira/go-xz', :commit => '0c531f070014e218b21f3cfca801cc992d52726d'
 gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
 gom 'github.com/smira/flag', :commit => '357ed3e599ffcbd4aeaa828e1d10da2df3ea5107'
 gom 'github.com/smira/go-ftp-protocol/protocol', :commit => '066b75c2b70dca7ae10b1b88b47534a3c31ccfaa'
-gom 'github.com/syndtr/gosnappy/snappy', :commit => 'ce8acff4829e0c2458a67ead32390ac0a381c862'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => '97e257099d2ab9578151ba85e2641e2cd14d3ca8'
+gom 'github.com/smira/go-uuid/uuid', :commit => 'ed3ca8a15a931b141440a7e98e4f716eec255f7d'
+gom 'github.com/smira/lzma', :commit => '2a7c55cad4a2d02ab972a03357db5760833a49bc'
+gom 'github.com/golang/snappy', :commit => '723cc1e459b8eea2dea4583200fd60757d40097a'
+gom 'github.com/syndtr/goleveldb/leveldb', :commit => '1a9d62f03ea92815b46fcaab357cfd4df264b1a0'
 gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
 gom 'github.com/vaughan0/go-ini', :commit => 'a98ad7ee00ec53921f08832bc06ecf7fd600e6a1'
 gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2013-2014 Andrey Smirnov. All rights reserved.
+Copyright 2013-2015 aptly authors. All rights reserved.
 
 MIT License
 

Makefile

@@ -67,7 +67,8 @@ package:
 	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
 	gzip root/usr/share/man/man1/aptly.1
 	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
-	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 --deb-recommends graphviz -C root/ .
+	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 \
+	   --deb-recommends graphviz --deb-recommends xz-utils -C root/ .
 	mv aptly_$(VERSION)_*.deb ~
 
 src-package:

README.rst

@@ -64,7 +64,7 @@ If you would like to use nightly builds (unstable), please use following reposit
 
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.3+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.4+ required)::
 
     go get -u github.com/mattn/gom
     mkdir -p $GOPATH/src/github.com/smira/aptly
@@ -78,4 +78,29 @@ should work as well, but might fail or produce different result (if external lib
 
 If you don't have Go installed (or older version), you can easily install Go using `gvm <https://github.com/moovweb/gvm/>`_.
 
+Integrations
+------------
 
+Vagrant:
+
+-   `Vagrant configuration <https://github.com/sepulworld/aptly-vagrant>`_ by
+    Zane Williamson, allowing to bring two virtual servers, one with aptly installed
+    and another one set up to install packages from repository published by aptly
+
+Docker:
+
+-    `Docker container <https://github.com/mikepurvis/aptly-docker>`_ with aptly inside by Mike Purvis
+
+With configuration management systems:
+
+-   `Chef cookbook <https://github.com/hw-cookbooks/aptly>`_ by Aaron Baer
+    (Heavy Water Operations, LLC)
+-   `Puppet module <https://github.com/alphagov/puppet-aptly>`_ by
+    Government Digital Services
+-   `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by
+    Forrest Alvarez and Brian Jackson
+-   `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine
+
+CLI for aptly API:
+
+-   `Ruby aptly CLI/library <https://github.com/sepulworld/aptly_cli>`_ by Zane Williamson

api/api.go

@@ -22,35 +22,84 @@ func apiVersion(c *gin.Context) {
 	c.JSON(200, gin.H{"Version": aptly.Version})
 }
 
-// Periodically flushes CollectionFactory to free up memory used by collections,
-// flushing caches.
+const (
+	ACQUIREDB = iota
+	RELEASEDB
+)
+
+// Periodically flushes CollectionFactory to free up memory used by
+// collections, flushing caches. If the two channels are provided,
+// they are used to acquire and release the database.
 //
 // Should be run in goroutine!
-func cacheFlusher() {
+func cacheFlusher(requests chan int, acks chan error) {
 	ticker := time.Tick(15 * time.Minute)
 
 	for {
 		<-ticker
 
-		// lock everything to eliminate in-progress calls
-		r := context.CollectionFactory().RemoteRepoCollection()
-		r.Lock()
-		defer r.Unlock()
+		func() {
+			// lock database if needed
+			if requests != nil {
+				requests <- ACQUIREDB
+				err := <-acks
+				if err != nil {
+					return
+				}
+				defer func() {
+					requests <- RELEASEDB
+					<-acks
+				}()
+			}
 
-		l := context.CollectionFactory().LocalRepoCollection()
-		l.Lock()
-		defer l.Unlock()
+			// lock everything to eliminate in-progress calls
+			r := context.CollectionFactory().RemoteRepoCollection()
+			r.Lock()
+			defer r.Unlock()
 
-		s := context.CollectionFactory().SnapshotCollection()
-		s.Lock()
-		defer s.Unlock()
+			l := context.CollectionFactory().LocalRepoCollection()
+			l.Lock()
+			defer l.Unlock()
 
-		p := context.CollectionFactory().PublishedRepoCollection()
-		p.Lock()
-		defer p.Unlock()
+			s := context.CollectionFactory().SnapshotCollection()
+			s.Lock()
+			defer s.Unlock()
 
-		// all collections locked, flush them
-		context.CollectionFactory().Flush()
+			p := context.CollectionFactory().PublishedRepoCollection()
+			p.Lock()
+			defer p.Unlock()
+
+			// all collections locked, flush them
+			context.CollectionFactory().Flush()
+		}()
+	}
+}
+
+// Acquire database lock and release it when not needed anymore. Two
+// channels must be provided. The first one is to receive requests to
+// acquire/release the database and the second one is to send acks.
+//
+// Should be run in a goroutine!
+func acquireDatabase(requests chan int, acks chan error) {
+	clients := 0
+	for {
+		request := <-requests
+		switch request {
+		case ACQUIREDB:
+			if clients == 0 {
+				acks <- context.ReOpenDatabase()
+			} else {
+				acks <- nil
+			}
+			clients++
+		case RELEASEDB:
+			clients--
+			if clients == 0 {
+				acks <- context.CloseDatabase()
+			} else {
+				acks <- nil
+			}
+		}
 	}
 }
 

api/repos.go

@@ -315,12 +315,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
 	}
 
-	packageFiles, failedFiles, err = deb.CollectPackageFiles(sources, reporter)
-
-	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to collect package files: %s", err))
-		return
-	}
+	packageFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
 
 	list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
 	if err != nil {
@@ -329,7 +324,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 	}
 
 	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), reporter)
+		context.CollectionFactory().PackageCollection(), reporter, nil)
 	failedFiles = append(failedFiles, failedFiles2...)
 
 	if err != nil {

api/router.go

@@ -12,11 +12,41 @@ var context *ctx.AptlyContext
 func Router(c *ctx.AptlyContext) http.Handler {
 	context = c
 
-	go cacheFlusher()
-
 	router := gin.Default()
 	router.Use(gin.ErrorLogger())
 
+	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
+		// We use a goroutine to count the number of
+		// concurrent requests. When no more requests are
+		// running, we close the database to free the lock.
+		requests := make(chan int)
+		acks := make(chan error)
+
+		go acquireDatabase(requests, acks)
+		go cacheFlusher(requests, acks)
+
+		router.Use(func(c *gin.Context) {
+			requests <- ACQUIREDB
+			err := <-acks
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+			defer func() {
+				requests <- RELEASEDB
+				err = <-acks
+				if err != nil {
+					c.Fail(500, err)
+					return
+				}
+			}()
+			c.Next()
+		})
+
+	} else {
+		go cacheFlusher(nil, nil)
+	}
+
 	root := router.Group("/api")
 
 	{

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
 // Version of aptly
-const Version = "0.9.5"
+const Version = "0.9.6"
 
 // Enable debugging features?
 const EnableDebug = false

cmd/api_serve.go

@@ -46,6 +46,7 @@ Example:
 	}
 
 	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening")
+	cmd.Flag.Bool("no-lock", false, "don't lock the database")
 
 	return cmd
 

cmd/cmd.go

@@ -2,12 +2,14 @@
 package cmd
 
 import (
+	"bytes"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"os"
+	"text/template"
 	"time"
 )
 
@@ -34,6 +36,32 @@ func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 	return
 }
 
+// PrintPackageList shows package list with specified format or default representation
+func PrintPackageList(result *deb.PackageList, format string) error {
+	if format == "" {
+		return result.ForEach(func(p *deb.Package) error {
+			context.Progress().Printf("%s\n", p)
+			return nil
+		})
+	}
+
+	formatTemplate, err := template.New("format").Parse(format)
+	if err != nil {
+		return fmt.Errorf("error parsing -format template: %s", err)
+	}
+
+	return result.ForEach(func(p *deb.Package) error {
+		b := &bytes.Buffer{}
+		err = formatTemplate.Execute(b, p.ExtendedStanza())
+		if err != nil {
+			return fmt.Errorf("error applying template: %s", err)
+		}
+		context.Progress().Printf("%s\n", b.String())
+		return nil
+	})
+
+}
+
 // LookupOption checks boolean flag with default (usually config) and command-line
 // setting
 func LookupOption(defaultValue bool, flags *flag.FlagSet, name string) (result bool) {
@@ -83,7 +111,7 @@ package environment to new version.`,
 	cmd.Flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests")
 	cmd.Flag.Bool("dep-follow-source", false, "when processing dependencies, follow from binary to Source packages")
 	cmd.Flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends")
-	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if depdency is 'a|b'")
+	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
 	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
 

cmd/graph.go

@@ -4,11 +4,13 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
 )
 
 func aptlyGraph(cmd *commander.Command, args []string) error {
@@ -34,9 +36,16 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 	tempfile.Close()
 	os.Remove(tempfile.Name())
 
-	tempfilename := tempfile.Name() + ".png"
+	format := context.Flags().Lookup("format").Value.String()
+	output := context.Flags().Lookup("output").Value.String()
 
-	command := exec.Command("dot", "-Tpng", "-o"+tempfilename)
+	if filepath.Ext(output) != "" {
+		format = filepath.Ext(output)[1:]
+	}
+
+	tempfilename := tempfile.Name() + "." + format
+
+	command := exec.Command("dot", "-T"+format, "-o"+tempfilename)
 	command.Stderr = os.Stderr
 
 	stdin, err := command.StdinPipe()
@@ -64,10 +73,18 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	err = exec.Command("open", tempfilename).Run()
-	if err != nil {
-		fmt.Printf("Rendered to PNG file: %s\n", tempfilename)
-		err = nil
+	if output != "" {
+		err = utils.CopyFile(tempfilename, output)
+		if err != nil {
+			return fmt.Errorf("unable to copy %s -> %s: %s", tempfilename, output, err)
+		}
+		_ = os.Remove(tempfilename)
+
+		fmt.Printf("Output saved to %s\n", output)
+	} else {
+		fmt.Printf("Rendered to %s file: %s, trying to open it...\n", format, tempfilename)
+
+		_ = exec.Command("open", tempfilename).Run()
 	}
 
 	return err
@@ -89,5 +106,8 @@ Example:
 `,
 	}
 
+	cmd.Flag.String("format", "png", "render graph to specified format (png, svg, pdf, etc.)")
+	cmd.Flag.String("output", "", "specify output filename, default is to open result in viewer")
+
 	return cmd
 }

cmd/mirror_search.go

@@ -21,6 +21,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/package_search.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -25,10 +24,8 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("no results")
 	}
 
-	result.ForEach(func(p *deb.Package) error {
-		context.Progress().Printf("%s\n", p)
-		return nil
-	})
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format)
 
 	return err
 }
@@ -48,5 +45,7 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-package-search", flag.ExitOnError),
 	}
 
+	cmd.Flag.String("format", "", "custom format for result printing")
+
 	return cmd
 }

cmd/publish_repo.go

@@ -41,6 +41,7 @@ Example:
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")

cmd/publish_snapshot.go

@@ -116,8 +116,12 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
-	published.Origin = cmd.Flag.Lookup("origin").Value.String()
-	published.Label = cmd.Flag.Lookup("label").Value.String()
+	published.Origin = context.Flags().Lookup("origin").Value.String()
+	published.Label = context.Flags().Lookup("label").Value.String()
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
 
 	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
@@ -203,6 +207,7 @@ Example:
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")

cmd/publish_switch.go

@@ -90,6 +90,10 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 			"the same package pool.\n")
 	}
 
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
@@ -143,6 +147,7 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 

cmd/publish_update.go

@@ -54,6 +54,10 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 			"the same package pool.\n")
 	}
 
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
@@ -102,6 +106,7 @@ Example:
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd

cmd/repo.go

@@ -21,6 +21,7 @@ func makeCmdRepo() *commander.Command {
 			makeCmdRepoShow(),
 			makeCmdRepoRename(),
 			makeCmdRepoSearch(),
+			makeCmdRepoInclude(),
 		},
 	}
 }

cmd/repo_add.go

@@ -42,15 +42,12 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	var packageFiles, failedFiles []string
 
-	packageFiles, failedFiles, err = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
-	if err != nil {
-		return fmt.Errorf("unable to collect package files: %s", err)
-	}
+	packageFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
 
 	var processedFiles, failedFiles2 []string
 
 	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()})
+		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil)
 	failedFiles = append(failedFiles, failedFiles2...)
 	if err != nil {
 		return fmt.Errorf("unable to import package files: %s", err)

cmd/repo_create.go

@@ -18,6 +18,14 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
 	repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
 
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		repo.Uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+	}
+
 	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add local repo: %s", err)
@@ -47,6 +55,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "main", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_edit.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/AlekSi/pointer"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -23,16 +25,30 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	if context.Flags().Lookup("comment").Value.String() != "" {
-		repo.Comment = context.Flags().Lookup("comment").Value.String()
-	}
+	var uploadersFile *string
 
-	if context.Flags().Lookup("distribution").Value.String() != "" {
-		repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
-	}
+	context.Flags().Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "comment":
+			repo.Comment = flag.Value.String()
+		case "distribution":
+			repo.DefaultDistribution = flag.Value.String()
+		case "component":
+			repo.DefaultComponent = flag.Value.String()
+		case "uploaders-file":
+			uploadersFile = pointer.ToString(flag.Value.String())
+		}
+	})
 
-	if context.Flags().Lookup("component").Value.String() != "" {
-		repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
+	if uploadersFile != nil {
+		if *uploadersFile != "" {
+			repo.Uploaders, err = deb.NewUploadersFromFile(*uploadersFile)
+			if err != nil {
+				return err
+			}
+		} else {
+			repo.Uploaders = nil
+		}
 	}
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
@@ -63,6 +79,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_include.go

@@ -0,0 +1,234 @@
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"os"
+	"path/filepath"
+	"text/template"
+)
+
+func aptlyRepoInclude(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	verifier, err := getVerifier(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+	}
+
+	if verifier == nil {
+		verifier = &utils.GpgVerifier{}
+	}
+
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
+	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
+	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
+
+	repoTemplate, err := template.New("repo").Parse(context.Flags().Lookup("repo").Value.Get().(string))
+	if err != nil {
+		return fmt.Errorf("error parsing -repo template: %s", err)
+	}
+
+	uploaders := (*deb.Uploaders)(nil)
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+
+		for i := range uploaders.Rules {
+			uploaders.Rules[i].CompiledCondition, err = query.Parse(uploaders.Rules[i].Condition)
+			if err != nil {
+				return fmt.Errorf("error parsing query %s: %s", uploaders.Rules[i].Condition, err)
+			}
+		}
+	}
+
+	reporter := &aptly.ConsoleResultReporter{Progress: context.Progress()}
+
+	var changesFiles, failedFiles, processedFiles []string
+
+	changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)
+
+	for _, path := range changesFiles {
+		var changes *deb.Changes
+
+		changes, err = deb.NewChanges(path)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", path, err)
+			continue
+		}
+
+		err = changes.VerifyAndParse(acceptUnsigned, ignoreSignatures, verifier)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		err = changes.Prepare()
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		repoName := &bytes.Buffer{}
+		err = repoTemplate.Execute(repoName, changes.Stanza)
+		if err != nil {
+			return fmt.Errorf("error applying template to repo: %s", err)
+		}
+
+		context.Progress().Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
+
+		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(repoName.String())
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		currentUploaders := uploaders
+		if repo.Uploaders != nil {
+			currentUploaders = repo.Uploaders
+			for i := range currentUploaders.Rules {
+				currentUploaders.Rules[i].CompiledCondition, err = query.Parse(currentUploaders.Rules[i].Condition)
+				if err != nil {
+					return fmt.Errorf("error parsing query %s: %s", currentUploaders.Rules[i].Condition, err)
+				}
+			}
+		}
+
+		if currentUploaders != nil {
+			if err = currentUploaders.IsAllowed(changes); err != nil {
+				failedFiles = append(failedFiles, path)
+				reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
+					changes.ChangesName, changes.SignatureKeys, err)
+				changes.Cleanup()
+				continue
+			}
+		}
+
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to load repo: %s", err)
+		}
+
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to load packages: %s", err)
+		}
+
+		packageFiles, _ := deb.CollectPackageFiles([]string{changes.TempDir}, reporter)
+
+		var restriction deb.PackageQuery
+
+		restriction, err = changes.PackageQuery()
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		var processedFiles2, failedFiles2 []string
+
+		processedFiles2, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+			context.CollectionFactory().PackageCollection(), reporter, restriction)
+
+		if err != nil {
+			return fmt.Errorf("unable to import package files: %s", err)
+		}
+
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+		if err != nil {
+			return fmt.Errorf("unable to save: %s", err)
+		}
+
+		err = changes.Cleanup()
+		if err != nil {
+			return err
+		}
+
+		for _, file := range failedFiles2 {
+			failedFiles = append(failedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		for _, file := range processedFiles2 {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		processedFiles = append(processedFiles, path)
+	}
+
+	if !noRemoveFiles {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err := os.Remove(file)
+			if err != nil {
+				return fmt.Errorf("unable to remove file: %s", err)
+			}
+		}
+	}
+
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
+	return err
+}
+
+func makeCmdRepoInclude() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoInclude,
+		UsageLine: "include <file.changes>|<directory> ...",
+		Short:     "add packages to local repositories based on .changes files",
+		Long: `
+Command include looks for .changes files in list of arguments or specified directories. Each
+.changes file is verified, parsed, referenced files are put into separate temporary directory
+and added into local repository. Successfully imported files are removed by default.
+
+Additionally uploads could be restricted with <uploaders.json> file. Rules in this file control
+uploads based on GPG key ID of .changes file signature and queries on .changes file fields.
+
+Example:
+
+  $ aptly repo include -repo=foo-release incoming/
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-include", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("no-remove-files", false, "don't remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
+	cmd.Flag.String("repo", "{{.Distribution}}", "which repo should files go to, defaults to Distribution field of .changes file")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of .changes file signature")
+	cmd.Flag.Bool("accept-unsigned", false, "accept unsigned .changes files")
+	cmd.Flag.String("uploaders-file", "", "path to uploaders.json file")
+
+	return cmd
+}

cmd/repo_search.go

@@ -21,6 +21,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/repo_show.go

@@ -29,6 +29,9 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Comment: %s\n", repo.Comment)
 	fmt.Printf("Default Distribution: %s\n", repo.DefaultDistribution)
 	fmt.Printf("Default Component: %s\n", repo.DefaultComponent)
+	if repo.Uploaders != nil {
+		fmt.Printf("Uploaders: %s\n", repo.Uploaders)
+	}
 	fmt.Printf("Number of packages: %d\n", repo.NumPackages())
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)

cmd/run.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	ctx "github.com/smira/aptly/context"
 	"github.com/smira/commander"
+	"os"
 )
 
 // Run runs single command starting from root cmd with args, optionally initializing context
@@ -14,7 +15,7 @@ func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode
 			if !ok {
 				panic(r)
 			}
-			fmt.Println("ERROR:", fatal.Message)
+			fmt.Fprintln(os.Stderr, "ERROR:", fatal.Message)
 			returnCode = fatal.ReturnCode
 		}
 	}()

cmd/snapshot_search.go

@@ -100,10 +100,8 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		return fmt.Errorf("no results")
 	}
 
-	result.ForEach(func(p *deb.Package) error {
-		context.Progress().Printf("%s\n", p)
-		return nil
-	})
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format)
 
 	return err
 }
@@ -124,6 +122,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/snapshot_verify.go

@@ -31,25 +31,25 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 
 	packageList, err := deb.NewPackageListFromRefList(snapshots[0].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
-		fmt.Errorf("unable to load packages: %s", err)
+		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
 	sourcePackageList := deb.NewPackageList()
 	err = sourcePackageList.Append(packageList)
 	if err != nil {
-		fmt.Errorf("unable to merge sources: %s", err)
+		return fmt.Errorf("unable to merge sources: %s", err)
 	}
 
 	var pL *deb.PackageList
 	for i := 1; i < len(snapshots); i++ {
 		pL, err = deb.NewPackageListFromRefList(snapshots[i].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 		if err != nil {
-			fmt.Errorf("unable to load packages: %s", err)
+			return fmt.Errorf("unable to load packages: %s", err)
 		}
 
 		err = sourcePackageList.Append(pL)
 		if err != nil {
-			fmt.Errorf("unable to merge sources: %s", err)
+			return fmt.Errorf("unable to merge sources: %s", err)
 		}
 	}
 

context/context.go

@@ -99,7 +99,7 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			}
 
 			if err != nil {
-				fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
+				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", configLocations[0])
 				utils.SaveConfig(configLocations[0], &utils.Config)
 			}
 		}
@@ -322,8 +322,8 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 
 			var err error
 			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
-				params.Region, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
-				params.EncryptionMethod, params.PlusWorkaround)
+				params.Region, params.Endpoint, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
+				params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel)
 			if err != nil {
 				Fatal(err)
 			}

database/leveldb.go

@@ -43,7 +43,8 @@ var (
 
 func internalOpen(path string) (*leveldb.DB, error) {
 	o := &opt.Options{
-		Filter: filter.NewBloomFilter(10),
+		Filter:                 filter.NewBloomFilter(10),
+		OpenFilesCacheCapacity: 256,
 	}
 
 	return leveldb.OpenFile(path, o)

deb/changes.go

@@ -0,0 +1,267 @@
+package deb
+
+import (
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// Changes is a result of .changes file parsing
+type Changes struct {
+	Changes               string
+	Distribution          string
+	Files                 PackageFiles
+	BasePath, ChangesName string
+	TempDir               string
+	Source                string
+	Binary                []string
+	Architectures         []string
+	Stanza                Stanza
+	SignatureKeys         []utils.GpgKey
+}
+
+// NewChanges moves .changes file into temporary directory and creates Changes structure
+func NewChanges(path string) (*Changes, error) {
+	var err error
+
+	c := &Changes{
+		BasePath:    filepath.Dir(path),
+		ChangesName: filepath.Base(path),
+	}
+
+	c.TempDir, err = ioutil.TempDir(os.TempDir(), "aptly")
+	if err != nil {
+		return nil, err
+	}
+
+	// copy .changes file into temporary directory
+	err = utils.CopyFile(filepath.Join(c.BasePath, c.ChangesName), filepath.Join(c.TempDir, c.ChangesName))
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// VerifyAndParse does optional signature verification and parses changes files
+func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier utils.Verifier) error {
+	input, err := os.Open(filepath.Join(c.TempDir, c.ChangesName))
+	if err != nil {
+		return err
+	}
+	defer input.Close()
+
+	isClearSigned, err := verifier.IsClearSigned(input)
+	if err != nil {
+		return err
+	}
+
+	input.Seek(0, 0)
+
+	if !isClearSigned && !acceptUnsigned {
+		return fmt.Errorf(".changes file is not signed and unsigned processing hasn't been enabled")
+	}
+
+	if isClearSigned && !ignoreSignature {
+		keyInfo, err := verifier.VerifyClearsigned(input, false)
+		if err != nil {
+			return err
+		}
+		input.Seek(0, 0)
+
+		c.SignatureKeys = keyInfo.GoodKeys
+	}
+
+	var text *os.File
+
+	if isClearSigned {
+		text, err = verifier.ExtractClearsigned(input)
+		if err != nil {
+			return err
+		}
+		defer text.Close()
+	} else {
+		text = input
+	}
+
+	reader := NewControlFileReader(text)
+	c.Stanza, err = reader.ReadStanza(false)
+	if err != nil {
+		return err
+	}
+
+	c.Distribution = c.Stanza["Distribution"]
+	c.Changes = c.Stanza["Changes"]
+	c.Source = c.Stanza["Source"]
+	c.Binary = strings.Fields(c.Stanza["Binary"])
+	c.Architectures = strings.Fields(c.Stanza["Architecture"])
+
+	c.Files, err = c.Files.ParseSumFields(c.Stanza)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Prepare creates temporary directory, copies file there and verifies checksums
+func (c *Changes) Prepare() error {
+	var err error
+
+	for _, file := range c.Files {
+		if filepath.Dir(file.Filename) != "." {
+			return fmt.Errorf("file is not in the same folder as .changes file: %s", file.Filename)
+		}
+
+		file.Filename = filepath.Base(file.Filename)
+
+		err = utils.CopyFile(filepath.Join(c.BasePath, file.Filename), filepath.Join(c.TempDir, file.Filename))
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, file := range c.Files {
+		var info utils.ChecksumInfo
+
+		info, err = utils.ChecksumsForFile(filepath.Join(c.TempDir, file.Filename))
+		if err != nil {
+			return err
+		}
+
+		if info.Size != file.Checksums.Size {
+			return fmt.Errorf("size mismatch: expected %v != obtained %v", file.Checksums.Size, info.Size)
+		}
+
+		if info.MD5 != file.Checksums.MD5 {
+			return fmt.Errorf("checksum mismatch MD5: expected %v != obtained %v", file.Checksums.MD5, info.MD5)
+		}
+
+		if info.SHA1 != file.Checksums.SHA1 {
+			return fmt.Errorf("checksum mismatch SHA1: expected %v != obtained %v", file.Checksums.SHA1, info.SHA1)
+		}
+
+		if info.SHA256 != file.Checksums.SHA256 {
+			return fmt.Errorf("checksum mismatch SHA256 expected %v != obtained %v", file.Checksums.SHA256, info.SHA256)
+		}
+	}
+
+	return nil
+}
+
+// Cleanup removes all temporary files
+func (c *Changes) Cleanup() error {
+	if c.TempDir == "" {
+		return nil
+	}
+
+	return os.RemoveAll(c.TempDir)
+}
+
+// PackageQuery returns query that every package should match to be included
+func (c *Changes) PackageQuery() (PackageQuery, error) {
+	var archQuery PackageQuery = &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: ""}
+	for _, arch := range c.Architectures {
+		archQuery = &OrQuery{L: &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: arch}, R: archQuery}
+	}
+
+	// if c.Source is empty, this would never match
+	sourceQuery := &AndQuery{
+		L: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"},
+		R: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Source},
+	}
+
+	var binaryQuery PackageQuery
+	if len(c.Binary) > 0 {
+		binaryQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Binary[0]}
+		for _, binary := range c.Binary[1:] {
+			binaryQuery = &OrQuery{
+				L: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: binary},
+				R: binaryQuery,
+			}
+		}
+
+		binaryQuery = &AndQuery{
+			L: &NotQuery{Q: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"}},
+			R: binaryQuery}
+	}
+
+	var nameQuery PackageQuery
+	if binaryQuery == nil {
+		nameQuery = sourceQuery
+	} else {
+		nameQuery = &OrQuery{L: sourceQuery, R: binaryQuery}
+	}
+
+	return &AndQuery{L: archQuery, R: nameQuery}, nil
+}
+
+// GetField implements PackageLike interface
+func (c *Changes) GetField(field string) string {
+	return c.Stanza[field]
+}
+
+// MatchesDependency implements PackageLike interface
+func (c *Changes) MatchesDependency(d Dependency) bool {
+	return false
+}
+
+// MatchesArchitecture implements PackageLike interface
+func (c *Changes) MatchesArchitecture(arch string) bool {
+	return false
+}
+
+// GetName implements PackageLike interface
+func (c *Changes) GetName() string {
+	return ""
+}
+
+// GetVersion implements PackageLike interface
+func (c *Changes) GetVersion() string {
+	return ""
+
+}
+
+// GetArchitecture implements PackageLike interface
+func (c *Changes) GetArchitecture() string {
+	return ""
+}
+
+// CollectChangesFiles walks filesystem collecting all .changes files
+func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (changesFiles, failedFiles []string) {
+	for _, location := range locations {
+		info, err2 := os.Stat(location)
+		if err2 != nil {
+			reporter.Warning("Unable to process %s: %s", location, err2)
+			failedFiles = append(failedFiles, location)
+			continue
+		}
+		if info.IsDir() {
+			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
+				if err3 != nil {
+					return err3
+				}
+				if info.IsDir() {
+					return nil
+				}
+
+				if strings.HasSuffix(info.Name(), ".changes") {
+					changesFiles = append(changesFiles, path)
+				}
+
+				return nil
+			})
+		} else if strings.HasSuffix(info.Name(), ".changes") {
+			changesFiles = append(changesFiles, location)
+		}
+	}
+
+	sort.Strings(changesFiles)
+
+	return
+}

deb/changes_test.go

@@ -0,0 +1,91 @@
+package deb
+
+import (
+	. "gopkg.in/check.v1"
+	"os"
+	"path/filepath"
+)
+
+type ChangesSuite struct {
+	Dir, Path string
+}
+
+var _ = Suite(&ChangesSuite{})
+
+func (s *ChangesSuite) SetUpTest(c *C) {
+	s.Dir = c.MkDir()
+	s.Path = filepath.Join(s.Dir, "calamares.changes")
+
+	f, err := os.Create(s.Path)
+	c.Assert(err, IsNil)
+
+	f.WriteString(changesFile)
+	f.Close()
+}
+
+func (s *ChangesSuite) TestParseAndVerify(c *C) {
+	changes, err := NewChanges(s.Path)
+	c.Assert(err, IsNil)
+
+	err = changes.VerifyAndParse(true, true, &NullVerifier{})
+	c.Check(err, IsNil)
+
+	c.Check(changes.Distribution, Equals, "sid")
+	c.Check(changes.Files, HasLen, 4)
+	c.Check(changes.Files[0].Filename, Equals, "calamares_0+git20141127.99.dsc")
+	c.Check(changes.Files[0].Checksums.Size, Equals, int64(1106))
+	c.Check(changes.Files[0].Checksums.MD5, Equals, "05fd8f3ffe8f362c5ef9bad2f936a56e")
+	c.Check(changes.Files[0].Checksums.SHA1, Equals, "79f10e955dab6eb25b7f7bae18213f367a3a0396")
+	c.Check(changes.Files[0].Checksums.SHA256, Equals, "35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc")
+	c.Check(changes.BasePath, Equals, s.Dir)
+	c.Check(changes.Architectures, DeepEquals, []string{"source", "amd64"})
+	c.Check(changes.Source, Equals, "calamares")
+	c.Check(changes.Binary, DeepEquals, []string{"calamares", "calamares-dbg"})
+}
+
+func (s *ChangesSuite) TestPackageQuery(c *C) {
+	changes, err := NewChanges(s.Path)
+	c.Assert(err, IsNil)
+
+	err = changes.VerifyAndParse(true, true, &NullVerifier{})
+	c.Check(err, IsNil)
+
+	q, err := changes.PackageQuery()
+	c.Check(err, IsNil)
+
+	c.Check(q.String(), Equals,
+		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), ((Name (= calamares-dbg)) | (Name (= calamares)))))")
+}
+
+var changesFile = `Format: 1.8
+Date: Thu, 27 Nov 2014 13:24:53 +0000
+Source: calamares
+Binary: calamares calamares-dbg
+Architecture: source amd64
+Version: 0+git20141127.99
+Distribution: sid
+Urgency: medium
+Maintainer: Rohan Garg <rohan@kde.org>
+Changed-By: Rohan <rohan@kde.org>
+Description:
+ calamares  - distribution-independent installer framework
+ calamares-dbg - distribution-independent installer framework -- debug symbols
+Changes:
+ calamares (0+git20141127.99) sid; urgency=medium
+ .
+   * Update from git
+Checksums-Sha1:
+ 79f10e955dab6eb25b7f7bae18213f367a3a0396 1106 calamares_0+git20141127.99.dsc
+ 294c28e2c8e34e72ca9ee0d9da5c14f3bf4188db 2694800 calamares_0+git20141127.99.tar.xz
+ d6c26c04b5407c7511f61cb3e3de60c4a1d6c4ff 1698924 calamares_0+git20141127.99_amd64.deb
+ a3da632d193007b0d4a1aff73159fde1b532d7a8 12835902 calamares-dbg_0+git20141127.99_amd64.deb
+Checksums-Sha256:
+ 35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc 1106 calamares_0+git20141127.99.dsc
+ 5576b9caaf814564830f95561227e4f04ee87b31da22c1371aab155cbf7ce395 2694800 calamares_0+git20141127.99.tar.xz
+ 2e6e2f232ed7ffe52369928ebdf5436d90feb37840286ffba79e87d57a43a2e9 1698924 calamares_0+git20141127.99_amd64.deb
+ 8dd926080ed7bad2e2439e37e49ce12d5f1357c5041b7da4d860a1041f878a8a 12835902 calamares-dbg_0+git20141127.99_amd64.deb
+Files:
+ 05fd8f3ffe8f362c5ef9bad2f936a56e 1106 devel optional calamares_0+git20141127.99.dsc
+ 097e55c81abd8e5f30bb2eed90c2c1e9 2694800 devel optional calamares_0+git20141127.99.tar.xz
+ 827fb3b12534241e119815d331e8197b 1698924 devel optional calamares_0+git20141127.99_amd64.deb
+ e6f8ce70f564d1f68cb57758b15b13e3 12835902 debug optional calamares-dbg_0+git20141127.99_amd64.deb`

deb/contents.go

@@ -0,0 +1,75 @@
+package deb
+
+import (
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"io"
+	"sort"
+	"strings"
+)
+
+// ContentsIndex calculates mapping from files to packages, with sorting and aggregation
+type ContentsIndex struct {
+	index map[string][]*Package
+}
+
+// NewContentsIndex creates empty ContentsIndex
+func NewContentsIndex() *ContentsIndex {
+	return &ContentsIndex{
+		index: make(map[string][]*Package),
+	}
+}
+
+// Push adds package to contents index, calculating package contents as required
+func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool) {
+	contents := p.Contents(packagePool)
+
+	for _, path := range contents {
+		index.index[path] = append(index.index[path], p)
+	}
+}
+
+// Empty checks whether index contains no packages
+func (index *ContentsIndex) Empty() bool {
+	return len(index.index) == 0
+}
+
+// WriteTo dumps sorted mapping of files to qualified package names
+func (index *ContentsIndex) WriteTo(w io.Writer) (int64, error) {
+	var n int64
+
+	paths := make([]string, len(index.index))
+
+	i := 0
+	for path := range index.index {
+		paths[i] = path
+		i++
+	}
+
+	sort.Strings(paths)
+
+	nn, err := fmt.Fprintf(w, "%s %s\n", "FILE", "LOCATION")
+	n += int64(nn)
+	if err != nil {
+		return n, err
+	}
+
+	for _, path := range paths {
+		packages := index.index[path]
+		parts := make([]string, 0, len(packages))
+		for i := range packages {
+			name := packages[i].QualifiedName()
+			if !utils.StrSliceHasItem(parts, name) {
+				parts = append(parts, name)
+			}
+		}
+		nn, err = fmt.Fprintf(w, "%s %s\n", path, strings.Join(parts, ","))
+		n += int64(nn)
+		if err != nil {
+			return n, err
+		}
+	}
+
+	return n, nil
+}

deb/deb.go

@@ -2,11 +2,13 @@ package deb
 
 import (
 	"archive/tar"
-	"bufio"
+	"compress/bzip2"
 	"compress/gzip"
 	"fmt"
 	"github.com/mkrautz/goar"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/go-xz"
+	"github.com/smira/lzma"
 	"io"
 	"os"
 	"strings"
@@ -24,16 +26,16 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	for {
 		header, err := library.Next()
 		if err == io.EOF {
-			return nil, fmt.Errorf("unable to find control.tar.gz part")
+			return nil, fmt.Errorf("unable to find control.tar.gz part in package %s", packageFile)
 		}
 		if err != nil {
-			return nil, fmt.Errorf("unable to read .deb archive: %s", err)
+			return nil, fmt.Errorf("unable to read .deb archive %s: %s", packageFile, err)
 		}
 
 		if header.Name == "control.tar.gz" {
 			ungzip, err := gzip.NewReader(library)
 			if err != nil {
-				return nil, fmt.Errorf("unable to ungzip: %s", err)
+				return nil, fmt.Errorf("unable to ungzip control file from %s. Error: %s", packageFile, err)
 			}
 			defer ungzip.Close()
 
@@ -41,15 +43,15 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 			for {
 				tarHeader, err := untar.Next()
 				if err == io.EOF {
-					return nil, fmt.Errorf("unable to find control file")
+					return nil, fmt.Errorf("unable to find control file in %s", packageFile)
 				}
 				if err != nil {
-					return nil, fmt.Errorf("unable to read .tar archive: %s", err)
+					return nil, fmt.Errorf("unable to read .tar archive from %s. Error: %s", packageFile, err)
 				}
 
 				if tarHeader.Name == "./control" || tarHeader.Name == "control" {
 					reader := NewControlFileReader(untar)
-					stanza, err := reader.ReadStanza()
+					stanza, err := reader.ReadStanza(false)
 					if err != nil {
 						return nil, err
 					}
@@ -69,16 +71,16 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 	}
 	defer file.Close()
 
-	line, err := bufio.NewReader(file).ReadString('\n')
+	isClearSigned, err := verifier.IsClearSigned(file)
+	file.Seek(0, 0)
+
 	if err != nil {
 		return nil, err
 	}
 
-	file.Seek(0, 0)
-
 	var text *os.File
 
-	if strings.Index(line, "BEGIN PGP SIGN") != -1 {
+	if isClearSigned {
 		text, err = verifier.ExtractClearsigned(file)
 		if err != nil {
 			return nil, err
@@ -89,7 +91,7 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 	}
 
 	reader := NewControlFileReader(text)
-	stanza, err := reader.ReadStanza()
+	stanza, err := reader.ReadStanza(false)
 	if err != nil {
 		return nil, err
 	}
@@ -97,3 +99,75 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 	return stanza, nil
 
 }
+
+// GetContentsFromDeb returns list of files installed by .deb package
+func GetContentsFromDeb(packageFile string) ([]string, error) {
+	file, err := os.Open(packageFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	library := ar.NewReader(file)
+	for {
+		header, err := library.Next()
+		if err == io.EOF {
+			return nil, fmt.Errorf("unable to find data.tar.* part in %s", packageFile)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("unable to read .deb archive from %s: %s", packageFile, err)
+		}
+
+		if strings.HasPrefix(header.Name, "data.tar") {
+			var tarInput io.Reader
+
+			switch header.Name {
+			case "data.tar":
+				tarInput = library
+			case "data.tar.gz":
+				ungzip, err := gzip.NewReader(library)
+				if err != nil {
+					return nil, fmt.Errorf("unable to ungzip data.tar.gz from %s: %s", packageFile,err)
+				}
+				defer ungzip.Close()
+				tarInput = ungzip
+			case "data.tar.bz2":
+				tarInput = bzip2.NewReader(library)
+			case "data.tar.xz":
+				unxz, err := xz.NewReader(library)
+				if err != nil {
+					return nil, fmt.Errorf("unable to unxz data.tar.xz from %s: %s", packageFile, err)
+				}
+				defer unxz.Close()
+				tarInput = unxz
+			case "data.tar.lzma":
+				unlzma := lzma.NewReader(library)
+				defer unlzma.Close()
+				tarInput = unlzma
+			default:
+				return nil, fmt.Errorf("unsupported tar compression in %s: %s", packageFile, header.Name)
+			}
+
+			untar := tar.NewReader(tarInput)
+			var results []string
+			for {
+				tarHeader, err := untar.Next()
+				if err == io.EOF {
+					return results, nil
+				}
+				if err != nil {
+					return nil, fmt.Errorf("unable to read .tar archive from %s: %s", packageFile, err)
+				}
+
+				if tarHeader.Typeflag == tar.TypeDir {
+					continue
+				}
+
+				if strings.HasPrefix(tarHeader.Name, "./") {
+					tarHeader.Name = tarHeader.Name[2:]
+				}
+				results = append(results, tarHeader.Name)
+			}
+		}
+	}
+}

deb/deb_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 type DebSuite struct {
-	debFile, dscFile, dscFileNoSign string
+	debFile, debFile2, dscFile, dscFileNoSign string
 }
 
 var _ = Suite(&DebSuite{})
@@ -17,6 +17,7 @@ var _ = Suite(&DebSuite{})
 func (s *DebSuite) SetUpSuite(c *C) {
 	_, _File, _, _ := runtime.Caller(0)
 	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	s.debFile2 = filepath.Join(filepath.Dir(_File), "../system/changes/hardlink_0.2.1_amd64.deb")
 	s.dscFile = filepath.Join(filepath.Dir(_File), "../system/files/pyspi_0.6.1-1.3.dsc")
 	s.dscFileNoSign = filepath.Join(filepath.Dir(_File), "../system/files/pyspi-0.6.1-1.3.stripped.dsc")
 }
@@ -27,7 +28,7 @@ func (s *DebSuite) TestGetControlFileFromDeb(c *C) {
 
 	_, _File, _, _ := runtime.Caller(0)
 	_, err = GetControlFileFromDeb(_File)
-	c.Check(err, ErrorMatches, "unable to read .deb archive: ar: missing global header")
+	c.Check(err, ErrorMatches, "^.+ar: missing global header")
 
 	st, err := GetControlFileFromDeb(s.debFile)
 	c.Check(err, IsNil)
@@ -55,3 +56,15 @@ func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
 	c.Check(st["Version"], Equals, "0.6.1-1.4")
 	c.Check(st["Source"], Equals, "pyspi")
 }
+
+func (s *DebSuite) TestGetContentsFromDeb(c *C) {
+	contents, err := GetContentsFromDeb(s.debFile)
+	c.Check(err, IsNil)
+	c.Check(contents, DeepEquals, []string{"usr/share/doc/libboost-program-options-dev/changelog.gz",
+		"usr/share/doc/libboost-program-options-dev/copyright"})
+
+	contents, err = GetContentsFromDeb(s.debFile2)
+	c.Check(err, IsNil)
+	c.Check(contents, DeepEquals, []string{"usr/bin/hardlink", "usr/share/man/man1/hardlink.1.gz",
+		"usr/share/doc/hardlink/changelog.gz", "usr/share/doc/hardlink/copyright", "usr/share/doc/hardlink/NEWS.Debian.gz"})
+}

deb/format.go

@@ -92,16 +92,42 @@ func (s Stanza) Copy() (result Stanza) {
 	return
 }
 
-// Write single field from Stanza to writer
-func writeField(w *bufio.Writer, field, value string) (err error) {
-	_, multiline := multilineFields[field]
+func isMultilineField(field string, isRelease bool) bool {
+	switch field {
+	case "Description":
+		return true
+	case "Files":
+		return true
+	case "Changes":
+		return true
+	case "Checksums-Sha1":
+		return true
+	case "Checksums-Sha256":
+		return true
+	case "Package-List":
+		return true
+	case "MD5Sum":
+		return isRelease
+	case "SHA1":
+		return isRelease
+	case "SHA256":
+		return isRelease
+	}
+	return false
+}
 
-	if !multiline {
+// Write single field from Stanza to writer
+func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) {
+	if !isMultilineField(field, isRelease) {
 		_, err = w.WriteString(field + ": " + value + "\n")
 	} else {
 		if !strings.HasSuffix(value, "\n") {
 			value = value + "\n"
 		}
+
+		if field != "Description" {
+			value = "\n" + value
+		}
 		_, err = w.WriteString(field + ":" + value)
 	}
 
@@ -122,7 +148,7 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 		value, ok := s[field]
 		if ok {
 			delete(s, field)
-			err := writeField(w, field, value)
+			err := writeField(w, field, value, isRelease)
 			if err != nil {
 				return err
 			}
@@ -130,7 +156,7 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 	}
 
 	for field, value := range s {
-		err := writeField(w, field, value)
+		err := writeField(w, field, value, isRelease)
 		if err != nil {
 			return err
 		}
@@ -144,20 +170,6 @@ var (
 	ErrMalformedStanza = errors.New("malformed stanza syntax")
 )
 
-var multilineFields = make(map[string]bool)
-
-func init() {
-	multilineFields["Description"] = true
-	multilineFields["Files"] = true
-	multilineFields["Changes"] = true
-	multilineFields["Checksums-Sha1"] = true
-	multilineFields["Checksums-Sha256"] = true
-	multilineFields["Package-List"] = true
-	multilineFields["SHA256"] = true
-	multilineFields["SHA1"] = true
-	multilineFields["MD5Sum"] = true
-}
-
 func canonicalCase(field string) string {
 	upper := strings.ToUpper(field)
 	switch upper {
@@ -198,7 +210,7 @@ func NewControlFileReader(r io.Reader) *ControlFileReader {
 }
 
 // ReadStanza reeads one stanza from control file
-func (c *ControlFileReader) ReadStanza() (Stanza, error) {
+func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
 	stanza := make(Stanza, 32)
 	lastField := ""
 	lastFieldMultiline := false
@@ -218,7 +230,7 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 			if lastFieldMultiline {
 				stanza[lastField] += line + "\n"
 			} else {
-				stanza[lastField] += strings.TrimSpace(line)
+				stanza[lastField] += " " + strings.TrimSpace(line)
 			}
 		} else {
 			parts := strings.SplitN(line, ":", 2)
@@ -226,7 +238,7 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 				return nil, ErrMalformedStanza
 			}
 			lastField = canonicalCase(parts[0])
-			_, lastFieldMultiline = multilineFields[lastField]
+			lastFieldMultiline = isMultilineField(lastField, isRelease)
 			if lastFieldMultiline {
 				stanza[lastField] = parts[1]
 				if parts[1] != "" {

deb/format_test.go

@@ -84,18 +84,18 @@ func (s *ControlFileSuite) SetUpTest(c *C) {
 func (s *ControlFileSuite) TestReadStanza(c *C) {
 	r := NewControlFileReader(s.reader)
 
-	stanza1, err := r.ReadStanza()
+	stanza1, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
-	stanza2, err := r.ReadStanza()
+	stanza2, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
-	stanza3, err := r.ReadStanza()
+	stanza3, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 	c.Assert(stanza3, IsNil)
 
 	c.Check(stanza1["Format"], Equals, "3.0 (quilt)")
-	c.Check(stanza1["Build-Depends"], Equals, "debhelper (>= 8),bash-completion (>= 1:1.1-3),libcurl4-nss-dev, libreadline-dev, libxml2-dev, libpcre3-dev, liboauth-dev, xsltproc, docbook-xsl, docbook-xml, dh-autoreconf")
+	c.Check(stanza1["Build-Depends"], Equals, "debhelper (>= 8), bash-completion (>= 1:1.1-3), libcurl4-nss-dev, libreadline-dev, libxml2-dev, libpcre3-dev, liboauth-dev, xsltproc, docbook-xsl, docbook-xml, dh-autoreconf")
 	c.Check(stanza1["Files"], Equals, " 3d5f65778bf3f89be03c313b0024b62c 1980 bti_032-1.dsc\n"+
 		" 1e0d0b693fdeebec268004ba41701baf 59773 bti_032.orig.tar.gz\n"+" ac1229a6d685023aeb8fcb0806324aa8 5065 bti_032-1.debian.tar.gz\n")
 	c.Check(len(stanza2), Equals, 20)
@@ -103,12 +103,12 @@ func (s *ControlFileSuite) TestReadStanza(c *C) {
 
 func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 	r := NewControlFileReader(s.reader)
-	stanza, err := r.ReadStanza()
+	stanza, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
 	buf := &bytes.Buffer{}
 	w := bufio.NewWriter(buf)
-	err = stanza.Copy().WriteTo(w, false, false)
+	err = stanza.Copy().WriteTo(w, true, false)
 	c.Assert(err, IsNil)
 	err = w.Flush()
 	c.Assert(err, IsNil)
@@ -116,7 +116,7 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 	str := buf.String()
 
 	r = NewControlFileReader(buf)
-	stanza2, err := r.ReadStanza()
+	stanza2, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
 	c.Assert(stanza2, DeepEquals, stanza)
@@ -140,7 +140,7 @@ func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
 		reader := bytes.NewBufferString(controlFile)
 		r := NewControlFileReader(reader)
 		for {
-			s, e := r.ReadStanza()
+			s, e := r.ReadStanza(false)
 			if s == nil && e == nil {
 				break
 			}

deb/graph.go

@@ -1,8 +1,8 @@
 package deb
 
 import (
-	"code.google.com/p/gographviz"
 	"fmt"
+	"github.com/awalterschulze/gographviz"
 	"strings"
 )
 

deb/import.go

@@ -10,7 +10,7 @@ import (
 )
 
 // CollectPackageFiles walks filesystem collecting all candidates for package files
-func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, failedFiles []string, err error) {
+func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, failedFiles []string) {
 	for _, location := range locations {
 		info, err2 := os.Stat(location)
 		if err2 != nil {
@@ -28,7 +28,7 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 				}
 
 				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
-					strings.HasSuffix(info.Name(), ".dsc") {
+					strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
 					packageFiles = append(packageFiles, path)
 				}
 
@@ -36,7 +36,7 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 			})
 		} else {
 			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
-				strings.HasSuffix(info.Name(), ".dsc") {
+				strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
 				packageFiles = append(packageFiles, location)
 			} else {
 				reporter.Warning("Unknown file extension: %s", location)
@@ -53,7 +53,7 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 
 // ImportPackageFiles imports files into local repository
 func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier utils.Verifier,
-	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter) (processedFiles []string, failedFiles []string, err error) {
+	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter, restriction PackageQuery) (processedFiles []string, failedFiles []string, err error) {
 	if forceReplace {
 		list.PrepareIndex()
 	}
@@ -150,6 +150,12 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
+		if restriction != nil && !restriction.Matches(p) {
+			reporter.Warning("%s has been ignored as it doesn't match restriction", p)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
 		err = collection.Update(p)
 		if err != nil {
 			reporter.Warning("Unable to save package %s: %s", p, err)

deb/index_files.go

@@ -24,6 +24,7 @@ type indexFile struct {
 	parent       *indexFiles
 	discardable  bool
 	compressable bool
+	onlyGzip     bool
 	signable     bool
 	relativePath string
 	tempFilename string
@@ -73,6 +74,9 @@ func (file *indexFile) Finalize(signer utils.Signer) error {
 	exts := []string{""}
 	if file.compressable {
 		exts = append(exts, ".gz", ".bz2")
+		if file.onlyGzip {
+			exts = []string{".gz"}
+		}
 	}
 
 	for _, ext := range exts {
@@ -215,6 +219,36 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 	return file
 }
 
+func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *indexFile {
+	if arch == "source" {
+		udeb = false
+	}
+	key := fmt.Sprintf("ci-%s-%s-%v", component, arch, udeb)
+	file, ok := files.indexes[key]
+	if !ok {
+		var relativePath string
+
+		if udeb {
+			relativePath = filepath.Join(component, fmt.Sprintf("Contents-udeb-%s", arch))
+		} else {
+			relativePath = filepath.Join(component, fmt.Sprintf("Contents-%s", arch))
+		}
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  true,
+			compressable: true,
+			onlyGzip:     true,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
 func (files *indexFiles) ReleaseFile() *indexFile {
 	return &indexFile{
 		parent:       files,

deb/list_test.go

@@ -379,6 +379,20 @@ func (s *PackageListSuite) TestFilter(c *C) {
 		&FieldQuery{Field: "$Architecture", Relation: VersionRegexp, Value: "i.*6", Regexp: regexp.MustCompile("i.*6")}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")},
+		&NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}},
+	}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}},
+		&FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")},
+	}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386")
 }
 
 func (s *PackageListSuite) TestVerifyDependencies(c *C) {

deb/local.go

@@ -2,9 +2,9 @@ package deb
 
 import (
 	"bytes"
-	"code.google.com/p/go-uuid/uuid"
 	"fmt"
 	"github.com/smira/aptly/database"
+	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
 	"log"
 	"sync"
@@ -22,6 +22,8 @@ type LocalRepo struct {
 	DefaultDistribution string `codec:",omitempty"`
 	// DefaultComponent
 	DefaultComponent string `codec:",omitempty"`
+	// Uploaders configuration
+	Uploaders *Uploaders `code:",omitempty" json:"-"`
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 }

deb/package.go

@@ -32,9 +32,10 @@ type Package struct {
 	// Is this >= 0.6 package?
 	V06Plus bool
 	// Offload fields
-	deps  *PackageDependencies
-	extra *Stanza
-	files *PackageFiles
+	deps     *PackageDependencies
+	extra    *Stanza
+	files    *PackageFiles
+	contents []string
 	// Mother collection
 	collection *PackageCollection
 }
@@ -114,62 +115,20 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 	delete(input, "Version")
 	delete(input, "Architecture")
 
+	var err error
+
 	files := make(PackageFiles, 0, 3)
-
-	parseSums := func(field string, setter func(sum *utils.ChecksumInfo, data string)) error {
-		for _, line := range strings.Split(input[field], "\n") {
-			line = strings.TrimSpace(line)
-			if line == "" {
-				continue
-			}
-			parts := strings.Fields(line)
-
-			if len(parts) != 3 {
-				return fmt.Errorf("unparseable hash sum line: %#v", line)
-			}
-
-			size, err := strconv.ParseInt(parts[1], 10, 64)
-			if err != nil {
-				return fmt.Errorf("unable to parse size: %s", err)
-			}
-
-			filename := filepath.Base(parts[2])
-
-			found := false
-			pos := 0
-			for i, file := range files {
-				if file.Filename == filename {
-					found = true
-					pos = i
-					break
-				}
-			}
-
-			if !found {
-				files = append(files, PackageFile{Filename: filename, downloadPath: input["Directory"]})
-				pos = len(files) - 1
-			}
-
-			files[pos].Checksums.Size = size
-			setter(&files[pos].Checksums, parts[0])
-		}
-
-		delete(input, field)
-
-		return nil
-	}
-
-	err := parseSums("Files", func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
+	files, err = files.ParseSumFields(input)
 	if err != nil {
 		return nil, err
 	}
-	err = parseSums("Checksums-Sha1", func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
-	if err != nil {
-		return nil, err
-	}
-	err = parseSums("Checksums-Sha256", func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
-	if err != nil {
-		return nil, err
+
+	delete(input, "Files")
+	delete(input, "Checksums-Sha1")
+	delete(input, "Checksums-Sha256")
+
+	for i := range files {
+		files[i].downloadPath = input["Directory"]
 	}
 
 	result.UpdateFiles(files)
@@ -211,14 +170,19 @@ func (p *Package) String() string {
 	return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.Architecture)
 }
 
-// MarshalJSON implements json.Marshaller interface
-func (p *Package) MarshalJSON() ([]byte, error) {
+// ExtendedStanza returns package stanza enhanced with aptly-specific fields
+func (p *Package) ExtendedStanza() Stanza {
 	stanza := p.Stanza()
 	stanza["FilesHash"] = fmt.Sprintf("%08x", p.FilesHash)
 	stanza["Key"] = string(p.Key(""))
 	stanza["ShortKey"] = string(p.ShortKey(""))
 
-	return json.Marshal(stanza)
+	return stanza
+}
+
+// MarshalJSON implements json.Marshaller interface
+func (p *Package) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.ExtendedStanza())
 }
 
 // GetField returns fields from package
@@ -336,6 +300,21 @@ func (p *Package) MatchesDependency(dep Dependency) bool {
 	panic("unknown relation")
 }
 
+// GetName returns package name
+func (p *Package) GetName() string {
+	return p.Name
+}
+
+// GetVersion returns package version
+func (p *Package) GetVersion() string {
+	return p.Version
+}
+
+// GetArchitecture returns package arch
+func (p *Package) GetArchitecture() string {
+	return p.Architecture
+}
+
 // GetDependencies compiles list of dependenices by flags from options
 func (p *Package) GetDependencies(options int) (dependencies []string) {
 	deps := p.Deps()
@@ -372,6 +351,16 @@ func (p *Package) GetDependencies(options int) (dependencies []string) {
 	return
 }
 
+// QualifiedName returns [$SECTION/]$NAME
+func (p *Package) QualifiedName() string {
+	section := p.Extra()["Section"]
+	if section != "" {
+		return section + "/" + p.Name
+	}
+
+	return p.Name
+}
+
 // Extra returns Stanza of extra fields (it may load it from collection)
 func (p *Package) Extra() Stanza {
 	if p.extra == nil {
@@ -410,6 +399,43 @@ func (p *Package) Files() PackageFiles {
 	return *p.files
 }
 
+// Contents returns cached package contents
+func (p *Package) Contents(packagePool aptly.PackagePool) []string {
+	if p.IsSource {
+		return nil
+	}
+
+	if p.contents == nil {
+		if p.collection == nil {
+			panic("contents == nil && collection == nil")
+		}
+
+		p.contents = p.collection.loadContents(p, packagePool)
+	}
+
+	return p.contents
+}
+
+// CalculateContents looks up contents in package file
+func (p *Package) CalculateContents(packagePool aptly.PackagePool) []string {
+	if p.IsSource {
+		return nil
+	}
+
+	file := p.Files()[0]
+	path, err := packagePool.Path(file.Filename, file.Checksums.MD5)
+	if err != nil {
+		panic(err)
+	}
+
+	contents, err := GetContentsFromDeb(path)
+	if err != nil {
+		panic(err)
+	}
+
+	return contents
+}
+
 // UpdateFiles saves new state of files
 func (p *Package) UpdateFiles(files PackageFiles) {
 	p.files = &files
@@ -456,10 +482,10 @@ func (p *Package) Stanza() (result Stanza) {
 			result["MD5sum"] = f.Checksums.MD5
 		}
 		if f.Checksums.SHA1 != "" {
-			result["SHA1"] = " " + f.Checksums.SHA1
+			result["SHA1"] = f.Checksums.SHA1
 		}
 		if f.Checksums.SHA256 != "" {
-			result["SHA256"] = " " + f.Checksums.SHA256
+			result["SHA256"] = f.Checksums.SHA256
 		}
 		result["Size"] = fmt.Sprintf("%d", f.Checksums.Size)
 	}

deb/package_collection.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"bytes"
 	"fmt"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/ugorji/go/codec"
 	"path/filepath"
@@ -10,9 +11,8 @@ import (
 
 // PackageCollection does management of packages in DB
 type PackageCollection struct {
-	db           database.Storage
-	encodeBuffer bytes.Buffer
-	codecHandle  *codec.MsgpackHandle
+	db          database.Storage
+	codecHandle *codec.MsgpackHandle
 }
 
 // Verify interface
@@ -161,45 +161,82 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 	return files
 }
 
+// loadContents loads or calculates and saves package contents
+func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.PackagePool) []string {
+	encoded, err := collection.db.Get(p.Key("xC"))
+	if err == nil {
+		contents := []string{}
+
+		decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
+		err = decoder.Decode(&contents)
+		if err != nil {
+			panic("unable to decode contents")
+		}
+
+		return contents
+	}
+
+	if err != database.ErrNotFound {
+		panic("unable to load contents")
+	}
+
+	contents := p.CalculateContents(packagePool)
+
+	var buf bytes.Buffer
+	err = codec.NewEncoder(&buf, collection.codecHandle).Encode(contents)
+	if err != nil {
+		panic("unable to encode contents")
+	}
+
+	err = collection.db.Put(p.Key("xC"), buf.Bytes())
+	if err != nil {
+		panic("unable to save contents")
+	}
+
+	return contents
+}
+
 // Update adds or updates information about package in DB checking for conficts first
 func (collection *PackageCollection) Update(p *Package) error {
-	encoder := codec.NewEncoder(&collection.encodeBuffer, collection.codecHandle)
+	var encodeBuffer bytes.Buffer
 
-	collection.encodeBuffer.Reset()
-	collection.encodeBuffer.WriteByte(0xc1)
-	collection.encodeBuffer.WriteByte(0x1)
+	encoder := codec.NewEncoder(&encodeBuffer, collection.codecHandle)
+
+	encodeBuffer.Reset()
+	encodeBuffer.WriteByte(0xc1)
+	encodeBuffer.WriteByte(0x1)
 	err := encoder.Encode(p)
 	if err != nil {
 		return err
 	}
 
-	err = collection.db.Put(p.Key(""), collection.encodeBuffer.Bytes())
+	err = collection.db.Put(p.Key(""), encodeBuffer.Bytes())
 	if err != nil {
 		return err
 	}
 
 	// Encode offloaded fields one by one
 	if p.files != nil {
-		collection.encodeBuffer.Reset()
+		encodeBuffer.Reset()
 		err = encoder.Encode(*p.files)
 		if err != nil {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xF"), collection.encodeBuffer.Bytes())
+		err = collection.db.Put(p.Key("xF"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
 	}
 
 	if p.deps != nil {
-		collection.encodeBuffer.Reset()
+		encodeBuffer.Reset()
 		err = encoder.Encode(*p.deps)
 		if err != nil {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xD"), collection.encodeBuffer.Bytes())
+		err = collection.db.Put(p.Key("xD"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
@@ -208,13 +245,13 @@ func (collection *PackageCollection) Update(p *Package) error {
 	}
 
 	if p.extra != nil {
-		collection.encodeBuffer.Reset()
+		encodeBuffer.Reset()
 		err = encoder.Encode(*p.extra)
 		if err != nil {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xE"), collection.encodeBuffer.Bytes())
+		err = collection.db.Put(p.Key("xE"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}

deb/package_deps.go

@@ -22,6 +22,12 @@ func parseDependencies(input Stanza, key string) []string {
 
 	delete(input, key)
 
+	value = strings.TrimSpace(value)
+	if value == "" {
+		// empty line is no depdencies
+		return nil
+	}
+
 	result := strings.Split(value, ",")
 	for i := range result {
 		result[i] = strings.TrimSpace(result[i])

deb/package_files.go

@@ -2,12 +2,15 @@ package deb
 
 import (
 	"encoding/binary"
+	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
 	"hash/fnv"
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
+	"strings"
 )
 
 // PackageFile is a single file entry in package
@@ -76,3 +79,65 @@ func (files PackageFiles) Swap(i, j int) {
 func (files PackageFiles) Less(i, j int) bool {
 	return files[i].Filename < files[j].Filename
 }
+
+func (files PackageFiles) parseSumField(input string, setter func(sum *utils.ChecksumInfo, data string)) (PackageFiles, error) {
+	for _, line := range strings.Split(input, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		parts := strings.Fields(line)
+
+		if len(parts) < 3 {
+			return nil, fmt.Errorf("unparseable hash sum line: %#v", line)
+		}
+
+		size, err := strconv.ParseInt(parts[1], 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("unable to parse size: %s", err)
+		}
+
+		filename := filepath.Base(parts[len(parts)-1])
+
+		found := false
+		pos := 0
+		for i, file := range files {
+			if file.Filename == filename {
+				found = true
+				pos = i
+				break
+			}
+		}
+
+		if !found {
+			files = append(files, PackageFile{Filename: filename})
+			pos = len(files) - 1
+		}
+
+		files[pos].Checksums.Size = size
+		setter(&files[pos].Checksums, parts[0])
+	}
+
+	return files, nil
+}
+
+// ParseSumFields populates PackageFiles by parsing stanza checksums fields
+func (files PackageFiles) ParseSumFields(stanza Stanza) (PackageFiles, error) {
+	var err error
+
+	files, err = files.parseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	files, err = files.parseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
+	if err != nil {
+		return nil, err
+	}
+	files, err = files.parseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	return files, nil
+}

deb/package_test.go

@@ -22,7 +22,7 @@ func (s *PackageSuite) SetUpTest(c *C) {
 	s.stanza = packageStanza.Copy()
 
 	buf := bytes.NewBufferString(sourcePackageMeta)
-	s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza()
+	s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza(false)
 }
 
 func (s *PackageSuite) TestNewFromPara(c *C) {
@@ -43,7 +43,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 }
 
 func (s *PackageSuite) TestNewUdebFromPara(c *C) {
-	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
 	p := NewUdebPackageFromControlFile(stanza)
 
 	c.Check(p.IsSource, Equals, false)
@@ -125,6 +125,10 @@ func (s *PackageSuite) TestStanza(c *C) {
 	p := NewPackageFromControlFile(s.stanza.Copy())
 	stanza := p.Stanza()
 
+	for k := range s.stanza {
+		c.Check(stanza[k], Equals, s.stanza[k])
+	}
+
 	c.Assert(stanza, DeepEquals, s.stanza)
 
 	p, _ = NewSourcePackageFromControlFile(s.sourceStanza.Copy())
@@ -152,7 +156,7 @@ func (s *PackageSuite) TestGetField(c *C) {
 
 	p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
 
-	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
 	p5 := NewUdebPackageFromControlFile(stanza5)
 
 	c.Check(p.GetField("$Source"), Equals, "alien-arena")
@@ -445,7 +449,7 @@ func (s *PackageSuite) TestVerifyFiles(c *C) {
 	c.Check(result, Equals, true)
 }
 
-var packageStanza = Stanza{"Source": "alien-arena", "Pre-Depends": "dpkg (>= 1.6)", "Suggests": "alien-arena-mars", "Recommends": "aliean-arena-luna", "Depends": "libc6 (>= 2.7), alien-arena-data (>= 7.40)", "Filename": "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", "SHA1": " 46955e48cad27410a83740a21d766ce362364024", "SHA256": " eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", "Priority": "extra", "Maintainer": "Debian Games Team <pkg-games-devel@lists.alioth.debian.org>", "Description": "Common files for Alien Arena client and server ALIEN ARENA is a standalone 3D first person online deathmatch shooter\n crafted from the original source code of Quake II and Quake III, released\n by id Software under the GPL license. With features including 32 bit\n graphics, new particle engine and effects, light blooms, reflective water,\n hi resolution textures and skins, hi poly models, stain maps, ALIEN ARENA\n pushes the envelope of graphical beauty rivaling today's top games.\n .\n This package installs the common files for Alien Arena.\n", "Homepage": "http://red.planetarena.org", "Tag": "role::app-data, role::shared-lib, special::auto-inst-parts", "Installed-Size": "456", "Version": "7.40-2", "Replaces": "alien-arena (<< 7.33-1)", "Size": "187518", "MD5sum": "1e8cba92c41420aa7baa8a5718d67122", "Package": "alien-arena-common", "Section": "contrib/games", "Architecture": "i386"}
+var packageStanza = Stanza{"Source": "alien-arena", "Pre-Depends": "dpkg (>= 1.6)", "Suggests": "alien-arena-mars", "Recommends": "aliean-arena-luna", "Depends": "libc6 (>= 2.7), alien-arena-data (>= 7.40)", "Filename": "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", "SHA1": "46955e48cad27410a83740a21d766ce362364024", "SHA256": "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", "Priority": "extra", "Maintainer": "Debian Games Team <pkg-games-devel@lists.alioth.debian.org>", "Description": "Common files for Alien Arena client and server ALIEN ARENA is a standalone 3D first person online deathmatch shooter\n crafted from the original source code of Quake II and Quake III, released\n by id Software under the GPL license. With features including 32 bit\n graphics, new particle engine and effects, light blooms, reflective water,\n hi resolution textures and skins, hi poly models, stain maps, ALIEN ARENA\n pushes the envelope of graphical beauty rivaling today's top games.\n .\n This package installs the common files for Alien Arena.\n", "Homepage": "http://red.planetarena.org", "Tag": "role::app-data, role::shared-lib, special::auto-inst-parts", "Installed-Size": "456", "Version": "7.40-2", "Replaces": "alien-arena (<< 7.33-1)", "Size": "187518", "MD5sum": "1e8cba92c41420aa7baa8a5718d67122", "Package": "alien-arena-common", "Section": "contrib/games", "Architecture": "i386"}
 
 const sourcePackageMeta = `Package: access-modifier-checker
 Binary: libaccess-modifier-checker-java, libaccess-modifier-checker-java-doc

deb/publish.go

@@ -3,12 +3,12 @@ package deb
 import (
 	"bufio"
 	"bytes"
-	"code.google.com/p/go-uuid/uuid"
 	"encoding/json"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
 	"io/ioutil"
 	"log"
@@ -43,6 +43,8 @@ type PublishedRepo struct {
 	Architectures []string
 	// SourceKind is "local"/"repo"
 	SourceKind string
+	// Skip contents generation
+	SkipContents bool
 
 	// Map of sources by each component: component name -> source UUID
 	Sources map[string]string
@@ -525,6 +527,8 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 		list.PrepareIndex()
 
+		contentIndexes := map[string]*ContentsIndex{}
+
 		err = list.ForEachIndexed(func(pkg *Package) error {
 			if progress != nil {
 				progress.AddBar(1)
@@ -550,6 +554,19 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				if pkg.MatchesArchitecture(arch) {
 					var bufWriter *bufio.Writer
 
+					if !p.SkipContents {
+						key := fmt.Sprintf("%s-%v", arch, pkg.IsUdeb)
+
+						contentIndex := contentIndexes[key]
+
+						if contentIndex == nil {
+							contentIndex = NewContentsIndex()
+							contentIndexes[key] = contentIndex
+						}
+
+						contentIndex.Push(pkg, packagePool)
+					}
+
 					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
 					if err != nil {
 						return err
@@ -569,6 +586,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			pkg.files = nil
 			pkg.deps = nil
 			pkg.extra = nil
+			pkg.contents = nil
 
 			return nil
 		})
@@ -577,6 +595,25 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			return fmt.Errorf("unable to process packages: %s", err)
 		}
 
+		for _, arch := range p.Architectures {
+			for _, udeb := range []bool{true, false} {
+				index := contentIndexes[fmt.Sprintf("%s-%v", arch, udeb)]
+				if index == nil || index.Empty() {
+					continue
+				}
+
+				bufWriter, err := indexes.ContentsIndex(component, arch, udeb).BufWriter()
+				if err != nil {
+					return fmt.Errorf("unable to generate contents index: %v", err)
+				}
+
+				_, err = index.WriteTo(bufWriter)
+				if err != nil {
+					return fmt.Errorf("unable to generate contents index: %v", err)
+				}
+			}
+		}
+
 		if progress != nil {
 			progress.ShutdownBar()
 		}
@@ -629,9 +666,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{"source"}), " ")
 	release["Description"] = " Generated by aptly\n"
-	release["MD5Sum"] = "\n"
-	release["SHA1"] = "\n"
-	release["SHA256"] = "\n"
+	release["MD5Sum"] = ""
+	release["SHA1"] = ""
+	release["SHA256"] = ""
 
 	release["Components"] = strings.Join(p.Components(), " ")
 

deb/publish_test.go

@@ -117,14 +117,19 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.packageCollection.Update(s.p3)
 
 	s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	s.repo.SkipContents = true
 
 	s.repo2, _ = NewPublishedRepo("", "ppa", "maverick", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo2.SkipContents = true
 
 	s.repo3, _ = NewPublishedRepo("", "linux", "natty", nil, []string{"main", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	s.repo3.SkipContents = true
 
 	s.repo4, _ = NewPublishedRepo("", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo4.SkipContents = true
 
 	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo5.SkipContents = true
 
 	poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
 	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
@@ -300,7 +305,7 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	c.Assert(err, IsNil)
 
 	cfr := NewControlFileReader(rf)
-	st, err := cfr.ReadStanza()
+	st, err := cfr.ReadStanza(true)
 	c.Assert(err, IsNil)
 
 	c.Check(st["Origin"], Equals, "ppa squeeze")
@@ -313,13 +318,13 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	cfr = NewControlFileReader(pf)
 
 	for i := 0; i < 3; i++ {
-		st, err = cfr.ReadStanza()
+		st, err = cfr.ReadStanza(false)
 		c.Assert(err, IsNil)
 
 		c.Check(st["Filename"], Equals, "pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
 	}
 
-	st, err = cfr.ReadStanza()
+	st, err = cfr.ReadStanza(false)
 	c.Assert(err, IsNil)
 	c.Assert(st, IsNil)
 
@@ -327,7 +332,7 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	c.Assert(err, IsNil)
 
 	cfr = NewControlFileReader(drf)
-	st, err = cfr.ReadStanza()
+	st, err = cfr.ReadStanza(true)
 	c.Assert(err, IsNil)
 
 	c.Check(st["Archive"], Equals, "squeeze")

deb/query.go

@@ -7,6 +7,16 @@ import (
 	"strings"
 )
 
+// PackageLike is something like Package :) To be refined later
+type PackageLike interface {
+	GetField(string) string
+	MatchesDependency(Dependency) bool
+	MatchesArchitecture(string) bool
+	GetName() string
+	GetVersion() string
+	GetArchitecture() string
+}
+
 // PackageCatalog is abstraction on top of PackageCollection and PackageList
 type PackageCatalog interface {
 	Scan(q PackageQuery) (result *PackageList)
@@ -18,7 +28,7 @@ type PackageCatalog interface {
 // PackageQuery is interface of predicate on Package
 type PackageQuery interface {
 	// Matches calculates match of condition against package
-	Matches(pkg *Package) bool
+	Matches(pkg PackageLike) bool
 	// Fast returns if search strategy is possible for this query
 	Fast(list PackageCatalog) bool
 	// Query performs search on package list
@@ -63,7 +73,7 @@ type DependencyQuery struct {
 }
 
 // Matches if any of L, R matches
-func (q *OrQuery) Matches(pkg *Package) bool {
+func (q *OrQuery) Matches(pkg PackageLike) bool {
 	return q.L.Matches(pkg) || q.R.Matches(pkg)
 }
 
@@ -89,7 +99,7 @@ func (q *OrQuery) String() string {
 }
 
 // Matches if both of L, R matches
-func (q *AndQuery) Matches(pkg *Package) bool {
+func (q *AndQuery) Matches(pkg PackageLike) bool {
 	return q.L.Matches(pkg) && q.R.Matches(pkg)
 }
 
@@ -120,7 +130,7 @@ func (q *AndQuery) String() string {
 }
 
 // Matches if not matches
-func (q *NotQuery) Matches(pkg *Package) bool {
+func (q *NotQuery) Matches(pkg PackageLike) bool {
 	return !q.Q.Matches(pkg)
 }
 
@@ -141,9 +151,9 @@ func (q *NotQuery) String() string {
 }
 
 // Matches on generic field
-func (q *FieldQuery) Matches(pkg *Package) bool {
+func (q *FieldQuery) Matches(pkg PackageLike) bool {
 	if q.Field == "$Version" {
-		return pkg.MatchesDependency(Dependency{Pkg: pkg.Name, Relation: q.Relation, Version: q.Value, Regexp: q.Regexp})
+		return pkg.MatchesDependency(Dependency{Pkg: pkg.GetName(), Relation: q.Relation, Version: q.Value, Regexp: q.Regexp})
 	}
 	if q.Field == "$Architecture" && q.Relation == VersionEqual {
 		return pkg.MatchesArchitecture(q.Value)
@@ -218,7 +228,7 @@ func (q *FieldQuery) String() string {
 }
 
 // Matches on dependency condition
-func (q *DependencyQuery) Matches(pkg *Package) bool {
+func (q *DependencyQuery) Matches(pkg PackageLike) bool {
 	return pkg.MatchesDependency(q.Dep)
 }
 
@@ -247,8 +257,8 @@ func (q *DependencyQuery) String() string {
 }
 
 // Matches on specific properties
-func (q *PkgQuery) Matches(pkg *Package) bool {
-	return pkg.Name == q.Pkg && pkg.Version == q.Version && pkg.Architecture == q.Arch
+func (q *PkgQuery) Matches(pkg PackageLike) bool {
+	return pkg.GetName() == q.Pkg && pkg.GetVersion() == q.Version && pkg.GetArchitecture() == q.Arch
 }
 
 // Fast is always true for package query

deb/remote.go

@@ -2,12 +2,12 @@ package deb
 
 import (
 	"bytes"
-	"code.google.com/p/go-uuid/uuid"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/http"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
 	"log"
 	"net/url"
@@ -263,7 +263,7 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error
 		}
 		defer inrelease.Close()
 
-		err = verifier.VerifyClearsigned(inrelease)
+		_, err = verifier.VerifyClearsigned(inrelease, true)
 		if err != nil {
 			goto splitsignature
 		}
@@ -304,7 +304,7 @@ ok:
 	defer release.Close()
 
 	sreader := NewControlFileReader(release)
-	stanza, err := sreader.ReadStanza()
+	stanza, err := sreader.ReadStanza(true)
 	if err != nil {
 		return err
 	}
@@ -443,7 +443,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 		sreader := NewControlFileReader(packagesReader)
 
 		for {
-			stanza, err := sreader.ReadStanza()
+			stanza, err := sreader.ReadStanza(false)
 			if err != nil {
 				return err
 			}

deb/remote_test.go

@@ -30,8 +30,8 @@ func (n *NullVerifier) VerifyDetachedSignature(signature, cleartext io.Reader) e
 	return nil
 }
 
-func (n *NullVerifier) VerifyClearsigned(clearsigned io.Reader) error {
-	return nil
+func (n *NullVerifier) VerifyClearsigned(clearsigned io.Reader, hint bool) (*utils.GpgKeyInfo, error) {
+	return nil, nil
 }
 
 func (n *NullVerifier) ExtractClearsigned(clearsigned io.Reader) (text *os.File, err error) {
@@ -43,6 +43,10 @@ func (n *NullVerifier) ExtractClearsigned(clearsigned io.Reader) (text *os.File,
 	return
 }
 
+func (n *NullVerifier) IsClearSigned(clearsign io.Reader) (bool, error) {
+	return false, nil
+}
+
 type PackageListMixinSuite struct {
 	p1, p2, p3 *Package
 	list       *PackageList
@@ -99,7 +103,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) {
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
 	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false)
-	c.Assert(err, ErrorMatches, ".*hexadecimal escape in host.*")
+	c.Assert(err, ErrorMatches, ".*(hexadecimal escape|percent-encoded characters) in host.*")
 }
 
 func (s *RemoteRepoSuite) TestFlatCreation(c *C) {
@@ -330,7 +334,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, false)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -363,7 +367,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, false)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 

deb/snapshot.go

@@ -2,11 +2,11 @@ package deb
 
 import (
 	"bytes"
-	"code.google.com/p/go-uuid/uuid"
 	"errors"
 	"fmt"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
 	"log"
 	"sort"

deb/uploaders.go

@@ -0,0 +1,105 @@
+package deb
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/DisposaBoy/JsonConfigReader"
+	"github.com/smira/aptly/utils"
+	"os"
+)
+
+// UploadersRule is single rule of format: what packages can group or key upload
+type UploadersRule struct {
+	Condition         string       `json:"condition"`
+	Allow             []string     `json:"allow"`
+	Deny              []string     `json:"deny"`
+	CompiledCondition PackageQuery `json:"-" codec:"-"`
+}
+
+func (u UploadersRule) String() string {
+	b, _ := json.Marshal(u)
+	return string(b)
+}
+
+// Uploaders is configuration of restrictions for .changes file importing
+type Uploaders struct {
+	Groups map[string][]string `json:"groups"`
+	Rules  []UploadersRule     `json:"rules"`
+}
+
+func (u *Uploaders) String() string {
+	b, _ := json.Marshal(u)
+	return string(b)
+}
+
+// NewUploadersFromFile loads Uploaders structue from .json file
+func NewUploadersFromFile(path string) (*Uploaders, error) {
+	uploaders := &Uploaders{}
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("error loading uploaders file: %s", err)
+	}
+	defer f.Close()
+
+	err = json.NewDecoder(JsonConfigReader.New(f)).Decode(&uploaders)
+	if err != nil {
+		return nil, fmt.Errorf("error loading uploaders file: %s", err)
+	}
+
+	return uploaders, nil
+}
+
+func (u *Uploaders) expandGroupsInternal(items []string, trail []string) []string {
+	result := []string{}
+
+	for _, item := range items {
+		// stop infinite recursion
+		if utils.StrSliceHasItem(trail, item) {
+			continue
+		}
+
+		group, ok := u.Groups[item]
+		if !ok {
+			result = append(result, item)
+		} else {
+			newTrail := append([]string(nil), trail...)
+			result = append(result, u.expandGroupsInternal(group, append(newTrail, item))...)
+		}
+	}
+
+	return result
+}
+
+// ExpandGroups expands list of keys/groups into list of keys
+func (u *Uploaders) ExpandGroups(items []string) []string {
+	result := u.expandGroupsInternal(items, []string{})
+
+	return utils.StrSliceDeduplicate(result)
+}
+
+// IsAllowed checks whether listed keys are allowed to upload given .changes file
+func (u *Uploaders) IsAllowed(changes *Changes) error {
+	for _, rule := range u.Rules {
+		if rule.CompiledCondition.Matches(changes) {
+			deny := u.ExpandGroups(rule.Deny)
+			for _, key := range changes.SignatureKeys {
+				for _, item := range deny {
+					if item == "*" || key.Matches(utils.GpgKey(item)) {
+						return fmt.Errorf("denied according to rule: %s", rule)
+					}
+				}
+			}
+
+			allow := u.ExpandGroups(rule.Allow)
+			for _, key := range changes.SignatureKeys {
+				for _, item := range allow {
+					if item == "*" || key.Matches(utils.GpgKey(item)) {
+						return nil
+					}
+				}
+			}
+		}
+	}
+
+	return fmt.Errorf("denied as no rule matches")
+}

deb/uploaders_test.go

@@ -0,0 +1,81 @@
+package deb
+
+import (
+	"github.com/smira/aptly/utils"
+	. "gopkg.in/check.v1"
+)
+
+type UploadersSuite struct {
+}
+
+var _ = Suite(&UploadersSuite{})
+
+func (s *UploadersSuite) TestExpandGroups(c *C) {
+	u := &Uploaders{
+		Groups: map[string][]string{
+			"group1": {"key1", "group2"},
+			"group2": {"key1", "key2", "key3", "group3"},
+			"group3": {},
+			"group4": {"key1", "group5"},
+			"group6": {"key1", "group8"},
+			"group7": {"key2", "group6"},
+			"group8": {"group7"},
+		},
+	}
+
+	c.Check(u.ExpandGroups([]string{"group1"}), DeepEquals, []string{"key1", "key2", "key3"})
+	c.Check(u.ExpandGroups([]string{"group2"}), DeepEquals, []string{"key1", "key2", "key3"})
+	c.Check(u.ExpandGroups([]string{"group3"}), DeepEquals, []string{})
+	c.Check(u.ExpandGroups([]string{"group4"}), DeepEquals, []string{"key1", "group5"})
+	c.Check(u.ExpandGroups([]string{"group6"}), DeepEquals, []string{"key1", "key2"})
+	c.Check(u.ExpandGroups([]string{"group7"}), DeepEquals, []string{"key2", "key1"})
+	c.Check(u.ExpandGroups([]string{"group8"}), DeepEquals, []string{"key2", "key1"})
+}
+
+func (s *UploadersSuite) TestIsAllowed(c *C) {
+	u := &Uploaders{
+		Groups: map[string][]string{
+			"group1": {"37E1C17570096AD1", "EC4B033C70096AD1"},
+		},
+		Rules: []UploadersRule{
+			{
+				CompiledCondition: &FieldQuery{Field: "Source", Relation: VersionEqual, Value: "calamares"},
+				Allow:             []string{"*"},
+			},
+			{
+				CompiledCondition: &FieldQuery{Field: "Source", Relation: VersionEqual, Value: "never-calamares"},
+				Deny:              []string{"*"},
+			},
+			{
+				CompiledCondition: &FieldQuery{Field: "Source", Relation: VersionEqual, Value: "some-calamares"},
+				Allow:             []string{"group1", "12345678"},
+			},
+			{
+				CompiledCondition: &FieldQuery{Field: "Source", Relation: VersionEqual, Value: "some-calamares"},
+				Deny:              []string{"45678901", "12345678"},
+			},
+		},
+	}
+
+	// no keys - not allowed
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{}, Stanza: Stanza{"Source": "calamares"}}), ErrorMatches, "denied as no rule matches")
+
+	// no rule - not allowed
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"37E1C17570096AD1", "EC4B033C70096AD1"}, Stanza: Stanza{"Source": "unknown-calamares"}}), ErrorMatches, "denied as no rule matches")
+
+	// first rule: allow anyone do stuff with calamares
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "calamares"}}), IsNil)
+
+	// second rule: nobody is allowed to do stuff with never-calamares
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "never-calamares"}}),
+		ErrorMatches, "denied according to rule: {\"condition\":\"\",\"allow\":null,\"deny\":\\[\"\\*\"\\]}")
+
+	// third rule: anyone from the group or explicit key
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"45678901", "12345678"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"37E1C17570096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"70096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+
+	// fourth rule: some are not allowed
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "45678901"}, Stanza: Stanza{"Source": "some-calamares"}}),
+		ErrorMatches, "denied according to rule: {\"condition\":\"\",\"allow\":null,\"deny\":\\[\"45678901\",\"12345678\"\\]}")
+}

http/download.go

@@ -1,10 +1,10 @@
 package http
 
 import (
-	"code.google.com/p/mxk/go1/flowcontrol"
 	"compress/bzip2"
 	"compress/gzip"
 	"fmt"
+	"github.com/mxk/go-flowrate/flowrate"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/go-ftp-protocol/protocol"
@@ -75,7 +75,7 @@ func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.
 	}
 
 	if downLimit > 0 {
-		downloader.aggWriter = flowcontrol.NewWriter(progress, downLimit)
+		downloader.aggWriter = flowrate.NewWriter(progress, downLimit)
 	} else {
 		downloader.aggWriter = progress
 	}
@@ -145,6 +145,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
 	}
+	req.Close = true
 
 	proxyURL, _ := downloader.client.Transport.(*http.Transport).Proxy(req)
 	if proxyURL == nil && (req.URL.Scheme == "http" || req.URL.Scheme == "https") {
@@ -326,6 +327,10 @@ func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChe
 		}
 
 		if !foundChecksum {
+			if !ignoreMismatch {
+				continue
+			}
+
 			file, err = DownloadTemp(downloader, tryURL)
 		}
 
@@ -344,5 +349,9 @@ func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChe
 
 		return uncompressed, file, err
 	}
+
+	if err == nil {
+		err = fmt.Errorf("no candidates for %s found", url)
+	}
 	return nil, nil, err
 }

http/download_test.go

@@ -257,27 +257,32 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
 	d.ExpectResponse("http://example.com/file.gz", "x")
-	r, file, err = DownloadTryCompression(d, "http://example.com/file", nil, false)
+	r, file, err = DownloadTryCompression(d, "http://example.com/file", nil, true)
 	c.Assert(err, ErrorMatches, "unexpected EOF")
 	c.Assert(d.Empty(), Equals, true)
 }
 
 func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) {
 	d := NewFakeDownloader()
-	_, _, err := DownloadTryCompression(d, "http://example.com/file", nil, false)
+	_, _, err := DownloadTryCompression(d, "http://example.com/file", nil, true)
 	c.Assert(err, ErrorMatches, "unexpected request.*")
 
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
 	d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
 	d.ExpectError("http://example.com/file", errors.New("403"))
-	_, _, err = DownloadTryCompression(d, "http://example.com/file", nil, false)
+	_, _, err = DownloadTryCompression(d, "http://example.com/file", nil, true)
 	c.Assert(err, ErrorMatches, "403")
 
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
 	d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
 	d.ExpectResponse("http://example.com/file", rawData)
-	_, _, err = DownloadTryCompression(d, "http://example.com/file", map[string]utils.ChecksumInfo{"file": {Size: 7}}, false)
+	expectedChecksums := map[string]utils.ChecksumInfo{
+		"file.bz2": {Size: 7},
+		"file.gz":  {Size: 7},
+		"file":     {Size: 7},
+	}
+	_, _, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false)
 	c.Assert(err, ErrorMatches, "checksums don't match.*")
 }

man/aptly.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "APTLY" "1" "March 2015" "" ""
+.TH "APTLY" "1" "January 2016" "" ""
 .
 .SH "NAME"
 \fBaptly\fR \- Debian repository management tool
@@ -52,13 +52,15 @@ Configuration file is stored in JSON format (default values shown below):
     "test": {
       "region": "us\-east\-1",
       "bucket": "repo",
+      "endpoint": "",
       "awsAccessKeyID": "",
       "awsSecretAccessKey": "",
       "prefix": "",
       "acl": "public\-read",
       "storageClass": "",
       "encryptionMethod": "",
-      "plusWorkaround": false
+      "plusWorkaround": false,
+      "disableMultiDel": false
     }
   },
   "SwiftPublishEndpoints": {
@@ -138,7 +140,7 @@ configuration of Amazon S3 publishing endpoints (see below)
 configuration of OpenStack Swift publishing endpoints (see below)
 .
 .SH "S3 PUBLISHING ENDPOINTS"
-aptly could be configured to publish repository directly to Amazon S3\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings:
+aptly could be configured to publish repository directly to Amazon S3 (or S3\-compatible cloud storage)\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings:
 .
 .TP
 \fBregion\fR
@@ -149,6 +151,10 @@ Amazon region for S3 bucket (e\.g\. \fBus\-east\-1\fR)
 bucket name
 .
 .TP
+\fBendpoint\fR
+(optional) when using S3\-compatible cloud storage, specify hostname of service endpoint here, region is ignored if endpoint is set (set region to some human\-readable name) (should be left blank for real Amazon S3)
+.
+.TP
 \fBprefix\fR
 (optional) do publishing under specified prefix in the bucket, defaults to no prefix (bucket root)
 .
@@ -170,7 +176,11 @@ bucket name
 .
 .TP
 \fBplusWorkaround\fR
-(optional) workaround misbehavior in apt and Amazon S3 for files with \fB+\fR in filename by creating two copies of package files with \fB+\fR in filename: one original and another one with spaces instead of plus signs With \fBplusWorkaround\fR enabled, package files with plus sign would be stored twice\. aptly might not cleanup files with spaces when published repository is dropped or updated (switched) to new version of repository (snapshot)\.
+(optional) workaround misbehavior in apt and Amazon S3 for files with \fB+\fR in filename by creating two copies of package files with \fB+\fR in filename: one original and another one with spaces instead of plus signs With \fBplusWorkaround\fR enabled, package files with plus sign would be stored twice\. aptly might not cleanup files with spaces when published repository is dropped or updated (switched) to new version of repository (snapshot)
+.
+.TP
+\fBdisableMultiDel\fR
+(optional) for S3\-compatible cloud storages which do not support \fBMultiDel\fR S3 API, enable this setting (file deletion would be slower with this setting enabled)
 .
 .P
 In order to publish to S3, specify endpoint as \fBs3:endpoint\-name:\fR before publishing prefix on the command line, e\.g\.:
@@ -304,6 +314,24 @@ When specified on command line, query may have to be quoted according to shell r
 .P
 \fBaptly repo import percona stable \(cqmysql\-client (>= 3\.6)\(cq\fR
 .
+.SH "PACKAGE DISPLAY FORMAT"
+Some aptly commands (\fBaptly mirror search\fR, \fBaptly package search\fR, \|\.\|\.\|\.) support \fB\-format\fR flag which allows to customize how search results are printed\. Golang templates are used to specify display format, with all package stanza fields available to template\. In addition to package stanza fields aptly provides:
+.
+.TP
+\fBKey\fR
+internal aptly package ID, unique for all packages in aptly (combination of \fBShortKey\fR and \fBFilesHash\fR)\.
+.
+.TP
+\fBFilesHash\fR
+hash that includes MD5 of all packages files\.
+.
+.TP
+\fBShortKey\fR
+package ID, which is unique in single list (mirror, repo, snapshot, \|\.\|\.\|\.), but not unique in whole aptly package collection\.
+.
+.P
+For example, default aptly display format could be presented with the following template: \fB{{\.Package}}_{{\.Version}}_{{\.Architecture}}\fR\. To display package name with dependencies: \fB{{\.Package}} | {{\.Depends}}\fR\. More information on Golang template syntax: http://godoc\.org/text/template
+.
 .SH "GLOBAL OPTIONS"
 .
 .TP
@@ -316,7 +344,7 @@ location of configuration file (default locations are /etc/aptly\.conf, ~/\.aptl
 .
 .TP
 \-\fBdep\-follow\-all\-variants\fR=false
-when processing dependencies, follow a & b if depdency is \(cqa|b\(cq
+when processing dependencies, follow a & b if dependency is \(cqa|b\(cq
 .
 .TP
 \-\fBdep\-follow\-recommends\fR=false
@@ -537,6 +565,10 @@ $ aptly mirror search wheezy\-main \(cq$Architecture (i386), Name (% *\-dev)\(cq
 Options:
 .
 .TP
+\-\fBformat\fR=
+custom format for result printing
+.
+.TP
 \-\fBwith\-deps\fR=false
 include dependencies into search results
 .
@@ -613,6 +645,10 @@ default component when publishing
 \-\fBdistribution\fR=
 default distribution when publishing
 .
+.TP
+\-\fBuploaders\-file\fR=
+uploaders\.json to be used when including \.changes into this repository
+.
 .SH "DELETE LOCAL REPOSITORY"
 \fBaptly\fR \fBrepo\fR \fBdrop\fR \fIname\fR
 .
@@ -659,6 +695,10 @@ default component when publishing
 \-\fBdistribution\fR=
 default distribution when publishing
 .
+.TP
+\-\fBuploaders\-file\fR=
+uploaders\.json to be used when including \.changes into this repository
+.
 .SH "IMPORT PACKAGES FROM MIRROR TO LOCAL REPOSITORY"
 \fBaptly\fR \fBrepo\fR \fBimport\fR \fIsrc\-mirror\fR \fIdst\-repo\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
 .
@@ -794,9 +834,59 @@ $ aptly repo search my\-software \(cq$Architecture (i386), Name (% *\-dev)\(cq
 Options:
 .
 .TP
+\-\fBformat\fR=
+custom format for result printing
+.
+.TP
 \-\fBwith\-deps\fR=false
 include dependencies into search results
 .
+.SH "ADD PACKAGES TO LOCAL REPOSITORIES BASED ON \.CHANGES FILES"
+\fBaptly\fR \fBrepo\fR \fBinclude\fR <file\.changes>|\fIdirectory\fR \fB\|\.\|\.\|\.\fR
+.
+.P
+Command include looks for \.changes files in list of arguments or specified directories\. Each \.changes file is verified, parsed, referenced files are put into separate temporary directory and added into local repository\. Successfully imported files are removed by default\.
+.
+.P
+Additionally uploads could be restricted with <uploaders\.json> file\. Rules in this file control uploads based on GPG key ID of \.changes file signature and queries on \.changes file fields\.
+.
+.P
+Example:
+.
+.P
+$ aptly repo include \-repo=foo\-release incoming/
+.
+.P
+Options:
+.
+.TP
+\-\fBaccept\-unsigned\fR=false
+accept unsigned \.changes files
+.
+.TP
+\-\fBforce\-replace\fR=false
+when adding package that conflicts with existing package, remove existing package
+.
+.TP
+\-\fBignore\-signatures\fR=false
+disable verification of \.changes file signature
+.
+.TP
+\-\fBkeyring\fR=
+gpg keyring to use when verifying Release file (could be specified multiple times)
+.
+.TP
+\-\fBno\-remove\-files\fR=false
+don\(cqt remove files that have been imported successfully into repository
+.
+.TP
+\-\fBrepo\fR={{\.Distribution}}
+which repo should files go to, defaults to Distribution field of \.changes file
+.
+.TP
+\-\fBuploaders\-file\fR=
+path to uploaders\.json file
+.
 .SH "CREATES SNAPSHOT OF MIRROR (LOCAL REPOSITORY) CONTENTS"
 \fBaptly\fR \fBsnapshot\fR \fBcreate\fR \fIname\fR \fBfrom\fR \fBmirror\fR \fImirror\-name\fR \fB|\fR \fBfrom\fR \fBrepo\fR \fIrepo\-name\fR \fB|\fR \fBempty\fR
 .
@@ -1038,6 +1128,10 @@ $ aptly snapshot search wheezy\-main \(cq$Architecture (i386), Name (% *\-dev)\(
 Options:
 .
 .TP
+\-\fBformat\fR=
+custom format for result printing
+.
+.TP
 \-\fBwith\-deps\fR=false
 include dependencies into search results
 .
@@ -1202,6 +1296,10 @@ GPG passhprase\-file for the key (warning: could be insecure)
 GPG secret keyring to use (instead of default)
 .
 .TP
+\-\fBskip\-contents\fR=false
+don\(cqt generate Contents indexes
+.
+.TP
 \-\fBskip\-signing\fR=false
 don\(cqt sign Release files with GPG
 .
@@ -1285,6 +1383,10 @@ GPG passhprase\-file for the key (warning: could be insecure)
 GPG secret keyring to use (instead of default)
 .
 .TP
+\-\fBskip\-contents\fR=false
+don\(cqt generate Contents indexes
+.
+.TP
 \-\fBskip\-signing\fR=false
 don\(cqt sign Release files with GPG
 .
@@ -1359,6 +1461,10 @@ GPG passhprase\-file for the key (warning: could be insecure)
 GPG secret keyring to use (instead of default)
 .
 .TP
+\-\fBskip\-contents\fR=false
+don\(cqt generate Contents indexes
+.
+.TP
 \-\fBskip\-signing\fR=false
 don\(cqt sign Release files with GPG
 .
@@ -1416,6 +1522,10 @@ GPG passhprase\-file for the key (warning: could be insecure)
 GPG secret keyring to use (instead of default)
 .
 .TP
+\-\fBskip\-contents\fR=false
+don\(cqt generate Contents indexes
+.
+.TP
 \-\fBskip\-signing\fR=false
 don\(cqt sign Release files with GPG
 .
@@ -1438,6 +1548,13 @@ $ aptly package search \(cq$Architecture (i386), Name (% *\-dev)\(cq
 .
 .IP "" 0
 .
+.P
+Options:
+.
+.TP
+\-\fBformat\fR=
+custom format for result printing
+.
 .SH "SHOW DETAILS ABOUT PACKAGES MATCHING QUERY"
 \fBaptly\fR \fBpackage\fR \fBshow\fR \fIpackage\-query\fR
 .
@@ -1522,6 +1639,29 @@ Options:
 \-\fBlisten\fR=:8080
 host:port for HTTP listening
 .
+.SH "START API HTTP SERVICE"
+\fBaptly\fR \fBapi\fR \fBserve\fR
+.
+.P
+Stat HTTP server with aptly REST API\.
+.
+.P
+Example:
+.
+.P
+$ aptly api serve \-listen=:8080
+.
+.P
+Options:
+.
+.TP
+\-\fBlisten\fR=:8080
+host:port for HTTP listening
+.
+.TP
+\-\fBno\-lock\fR=false
+don\(cqt lock the database
+.
 .SH "RENDER GRAPH OF RELATIONSHIPS"
 \fBaptly\fR \fBgraph\fR
 .
@@ -1534,6 +1674,17 @@ Example:
 .P
 $ aptly graph
 .
+.P
+Options:
+.
+.TP
+\-\fBformat\fR=png
+render graph to specified format (png, svg, pdf, etc\.)
+.
+.TP
+\-\fBoutput\fR=
+specify output filename, default is to open result in viewer
+.
 .SH "SHOW CURRENT APTLY\(cqS CONFIG"
 \fBaptly\fR \fBconfig\fR \fBshow\fR
 .
@@ -1577,6 +1728,18 @@ Options:
 \-\fBfilename\fR=
 specifies the filename that contains the commands to run
 .
+.SH "SHOW CURRENT APTLY\(cqS CONFIG"
+\fBaptly\fR \fBconfig\fR \fBshow\fR
+.
+.P
+Command show displays the current aptly configuration\.
+.
+.P
+Example:
+.
+.P
+$ aptly config show
+.
 .SH "ENVIRONMENT"
 If environment variable \fBHTTP_PROXY\fR is set \fBaptly\fR would use its value to proxy all HTTP requests\.
 .
@@ -1596,4 +1759,64 @@ general failure
 command parse failure
 .
 .SH "AUTHORS"
-Andrey Smirnov (me@smira\.ru)
+List of contributors, in chronological order:
+.
+.IP "\[ci]" 4
+Andrey Smirnov (https://github\.com/smira)
+.
+.IP "\[ci]" 4
+Sebastien Binet (https://github\.com/sbinet)
+.
+.IP "\[ci]" 4
+Ryan Uber (https://github\.com/ryanuber)
+.
+.IP "\[ci]" 4
+Simon Aquino (https://github\.com/queeno)
+.
+.IP "\[ci]" 4
+Vincent Batoufflet (https://github\.com/vbatoufflet)
+.
+.IP "\[ci]" 4
+Ivan Kurnosov (https://github\.com/zerkms)
+.
+.IP "\[ci]" 4
+Dmitrii Kashin (https://github\.com/freehck)
+.
+.IP "\[ci]" 4
+Chris Read (https://github\.com/cread)
+.
+.IP "\[ci]" 4
+Rohan Garg (https://github\.com/shadeslayer)
+.
+.IP "\[ci]" 4
+Russ Allbery (https://github\.com/rra)
+.
+.IP "\[ci]" 4
+Sylvain Baubeau (https://github\.com/lebauce)
+.
+.IP "\[ci]" 4
+Andrea Bernardo Ciddio (https://github\.com/bcandrea)
+.
+.IP "\[ci]" 4
+Michael Koval (https://github\.com/mkoval)
+.
+.IP "\[ci]" 4
+Alexander Guy (https://github\.com/alexanderguy)
+.
+.IP "\[ci]" 4
+Sebastien Badia (https://github\.com/sbadia)
+.
+.IP "\[ci]" 4
+Szymon Sobik (https://github\.com/sobczyk)
+.
+.IP "\[ci]" 4
+Paul Krohn (https://github\.com/paul\-krohn)
+.
+.IP "\[ci]" 4
+Vincent Bernat (https://github\.com/vincentbernat)
+.
+.IP "\[ci]" 4
+x539 (https://github\.com/x539)
+.
+.IP "" 0
+

man/aptly.1.ronn.tmpl

@@ -44,13 +44,15 @@ Configuration file is stored in JSON format (default values shown below):
         "test": {
           "region": "us-east-1",
           "bucket": "repo",
+          "endpoint": "",
           "awsAccessKeyID": "",
           "awsSecretAccessKey": "",
           "prefix": "",
           "acl": "public-read",
           "storageClass": "",
           "encryptionMethod": "",
-          "plusWorkaround": false
+          "plusWorkaround": false,
+          "disableMultiDel": false
         }
       },
       "SwiftPublishEndpoints": {
@@ -118,7 +120,8 @@ Options:
 
 ## S3 PUBLISHING ENDPOINTS
 
-aptly could be configured to publish repository directly to Amazon S3. First, publishing
+aptly could be configured to publish repository directly to Amazon S3 (or S3-compatible
+cloud storage). First, publishing
 endpoints should be described in aptly configuration file. Each endpoint has name
 and associated settings:
 
@@ -126,6 +129,10 @@ and associated settings:
      Amazon region for S3 bucket (e.g. `us-east-1`)
    * `bucket`:
      bucket name
+   * `endpoint`:
+     (optional) when using S3-compatible cloud storage, specify hostname of service endpoint here,
+     region is ignored if endpoint is set (set region to some human-readable name)
+     (should be left blank for real Amazon S3)
    * `prefix`:
      (optional) do publishing under specified prefix in the bucket, defaults to
      no prefix (bucket root)
@@ -152,7 +159,10 @@ and associated settings:
      and another one with spaces instead of plus signs
      With `plusWorkaround` enabled, package files with plus sign
      would be stored twice. aptly might not cleanup files with spaces when published
-     repository is dropped or updated (switched) to new version of repository (snapshot).
+     repository is dropped or updated (switched) to new version of repository (snapshot)
+   * `disableMultiDel`:
+     (optional) for S3-compatible cloud storages which do not support `MultiDel` S3 API,
+     enable this setting (file deletion would be slower with this setting enabled)
 
 In order to publish to S3, specify endpoint as `s3:endpoint-name:` before
 publishing prefix on the command line, e.g.:
@@ -264,6 +274,27 @@ When specified on command line, query may have to be quoted according to shell r
 
   `aptly repo import percona stable 'mysql-client (>= 3.6)'`
 
+## PACKAGE DISPLAY FORMAT
+
+Some aptly commands (`aptly mirror search`, `aptly package search`, ...) support `-format` flag
+which allows to customize how search results are printed. Golang templates are used to specify
+display format, with all package stanza fields available to template. In addition to package stanza
+fields aptly provides:
+
+ * `Key`:
+   internal aptly package ID, unique for all packages in aptly
+   (combination of `ShortKey` and `FilesHash`).
+
+ * `FilesHash`:
+   hash that includes MD5 of all packages files.
+
+ * `ShortKey`:
+   package ID, which is unique in single list (mirror, repo, snapshot, ...), but not unique
+   in whole aptly package collection.
+
+For example, default aptly display format could be presented with the following template:
+`{{"{{"}}.Package{{"}}"}}_{{"{{"}}.Version{{"}}"}}_{{"{{"}}.Architecture{{"}}"}}`. To display package name with dependencies:
+`{{"{{"}}.Package{{"}}"}} | {{"{{"}}.Depends{{"}}"}}`. More information on Golang template syntax: http://godoc.org/text/template
 
 ## GLOBAL OPTIONS
 
@@ -283,12 +314,16 @@ When specified on command line, query may have to be quoted according to shell r
 
 {{template "command" findCommand . "serve"}}
 
+{{template "command" findCommand . "api"}}
+
 {{template "command" findCommand . "graph"}}
 
 {{template "command" findCommand . "config"}}
 
 {{template "command" findCommand . "task"}}
 
+{{template "command" findCommand . "config"}}
+
 ## ENVIRONMENT
 
 If environment variable `HTTP_PROXY` is set `aptly` would use its value
@@ -309,7 +344,7 @@ to proxy all HTTP requests.
 
 ## AUTHORS
 
-Andrey Smirnov (me@smira.ru)
+{{authors}}
 
 {{end}}
 

man/gen.go

@@ -5,6 +5,7 @@ import (
 	"github.com/smira/aptly/cmd"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -43,6 +44,12 @@ func capitalize(s string) string {
 	return strings.Join(parts, " ")
 }
 
+var authorsS string
+
+func authors() string {
+	return authorsS
+}
+
 func main() {
 	command := cmd.RootCommand()
 	command.UsageLine = "aptly"
@@ -56,9 +63,24 @@ func main() {
 		"findCommand": findCommand,
 		"toUpper":     strings.ToUpper,
 		"capitalize":  capitalize,
+		"authors":     authors,
 	})
 	template.Must(templ.ParseFiles(filepath.Join(filepath.Dir(_File), "aptly.1.ronn.tmpl")))
 
+	authorsF, err := os.Open(filepath.Join(filepath.Dir(_File), "..", "AUTHORS"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsB, err := ioutil.ReadAll(authorsF)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsF.Close()
+
+	authorsS = string(authorsB)
+
 	output, err := os.Create(filepath.Join(filepath.Dir(_File), "aptly.1.ronn"))
 	if err != nil {
 		log.Fatal(err)

s3/public.go

@@ -6,6 +6,7 @@ import (
 	"github.com/mitchellh/goamz/s3"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/files"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
@@ -20,6 +21,8 @@ type PublishedStorage struct {
 	storageClass     string
 	encryptionMethod string
 	plusWorkaround   bool
+	disableMultiDel  bool
+	pathCache        map[string]string
 }
 
 // Check interface
@@ -29,7 +32,7 @@ var (
 
 // NewPublishedStorageRaw creates published storage from raw aws credentials
 func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL, prefix,
-	storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
+	storageClass, encryptionMethod string, plusWorkaround, disabledMultiDel bool) (*PublishedStorage, error) {
 	if defaultACL == "" {
 		defaultACL = "private"
 	}
@@ -44,7 +47,13 @@ func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL
 		prefix:           prefix,
 		storageClass:     storageClass,
 		encryptionMethod: encryptionMethod,
-		plusWorkaround:   plusWorkaround}
+		plusWorkaround:   plusWorkaround,
+		disableMultiDel:  disabledMultiDel,
+	}
+
+	result.s3.HTTPClient = func() *http.Client {
+		return RetryingClient
+	}
 	result.bucket = result.s3.Bucket(bucket)
 
 	return result, nil
@@ -52,19 +61,33 @@ func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL
 
 // NewPublishedStorage creates new instance of PublishedStorage with specified S3 access
 // keys, region and bucket name
-func NewPublishedStorage(accessKey, secretKey, region, bucket, defaultACL, prefix,
-	storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
+func NewPublishedStorage(accessKey, secretKey, region, endpoint, bucket, defaultACL, prefix,
+	storageClass, encryptionMethod string, plusWorkaround, disableMultiDel bool) (*PublishedStorage, error) {
 	auth, err := aws.GetAuth(accessKey, secretKey)
 	if err != nil {
 		return nil, err
 	}
 
-	awsRegion, ok := aws.Regions[region]
-	if !ok {
-		return nil, fmt.Errorf("unknown region: %#v", region)
+	var awsRegion aws.Region
+
+	if endpoint == "" {
+		var ok bool
+
+		awsRegion, ok = aws.Regions[region]
+		if !ok {
+			return nil, fmt.Errorf("unknown region: %#v", region)
+		}
+	} else {
+		awsRegion = aws.Region{
+			Name:                 region,
+			S3Endpoint:           endpoint,
+			S3LocationConstraint: true,
+			S3LowercaseBucket:    true,
+		}
 	}
 
-	return NewPublishedStorageRaw(auth, awsRegion, bucket, defaultACL, prefix, storageClass, encryptionMethod, plusWorkaround)
+	return NewPublishedStorageRaw(auth, awsRegion, bucket, defaultACL, prefix, storageClass, encryptionMethod,
+		plusWorkaround, disableMultiDel)
 }
 
 // String
@@ -123,6 +146,11 @@ func (storage *PublishedStorage) Remove(path string) error {
 	if err != nil {
 		return fmt.Errorf("error deleting %s from %s: %s", path, storage, err)
 	}
+
+	if storage.plusWorkaround && strings.Index(path, "+") != -1 {
+		// try to remove workaround version, but don't care about result
+		_ = storage.Remove(strings.Replace(path, "+", " ", -1))
+	}
 	return nil
 }
 
@@ -130,32 +158,38 @@ func (storage *PublishedStorage) Remove(path string) error {
 func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
 	const page = 1000
 
-	filelist, err := storage.Filelist(path)
+	filelist, _, err := storage.internalFilelist(path, false)
 	if err != nil {
 		return err
 	}
 
-	numParts := (len(filelist) + page - 1) / page
-
-	for i := 0; i < numParts; i++ {
-		var part []string
-		if i == numParts-1 {
-			part = filelist[i*page:]
-		} else {
-			part = filelist[i*page : (i+1)*page]
+	if storage.disableMultiDel {
+		for i := range filelist {
+			err = storage.bucket.Del(filepath.Join(storage.prefix, path, filelist[i]))
+			if err != nil {
+				return fmt.Errorf("error deleting path %s from %s: %s", filelist[i], storage, err)
+			}
 		}
-		paths := make([]string, len(part))
+	} else {
+		numParts := (len(filelist) + page - 1) / page
 
-		for i := range part {
-			paths[i] = filepath.Join(storage.prefix, path, part[i])
-		}
+		for i := 0; i < numParts; i++ {
+			var part []string
+			if i == numParts-1 {
+				part = filelist[i*page:]
+			} else {
+				part = filelist[i*page : (i+1)*page]
+			}
+			paths := make([]string, len(part))
 
-		err = storage.bucket.MultiDel(paths)
-		if err != nil {
-			return fmt.Errorf("error deleting multiple paths from %s: %s", storage, err)
-		}
-		if err != nil {
-			return err
+			for i := range part {
+				paths[i] = filepath.Join(storage.prefix, path, part[i])
+			}
+
+			err = storage.bucket.MultiDel(paths)
+			if err != nil {
+				return fmt.Errorf("error deleting multiple paths from %s: %s", storage, err)
+			}
 		}
 	}
 
@@ -179,17 +213,25 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 	poolPath := filepath.Join(storage.prefix, relPath)
 
 	var (
-		dstKey *s3.Key
-		err    error
+		err error
 	)
 
-	dstKey, err = storage.bucket.GetKey(poolPath)
-	if err != nil {
-		if s3err, ok := err.(*s3.Error); !ok || s3err.StatusCode != 404 {
-			return fmt.Errorf("error getting information about %s from %s: %s", poolPath, storage, err)
+	if storage.pathCache == nil {
+		paths, md5s, err := storage.internalFilelist(storage.prefix, true)
+		if err != nil {
+			return fmt.Errorf("error caching paths under prefix: %s", err)
 		}
-	} else {
-		destinationMD5 := strings.Replace(dstKey.ETag, "\"", "", -1)
+
+		storage.pathCache = make(map[string]string, len(paths))
+
+		for i := range paths {
+			storage.pathCache[paths[i]] = md5s[i]
+		}
+	}
+
+	destinationMD5, exists := storage.pathCache[relPath]
+
+	if exists {
 		if destinationMD5 == sourceMD5 {
 			return nil
 		}
@@ -200,12 +242,23 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 		}
 	}
 
-	return storage.PutFile(relPath, sourcePath)
+	err = storage.PutFile(relPath, sourcePath)
+	if err == nil {
+		storage.pathCache[relPath] = sourceMD5
+	}
+
+	return err
 }
 
 // Filelist returns list of files under prefix
 func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
-	result := []string{}
+	paths, _, err := storage.internalFilelist(prefix, true)
+	return paths, err
+}
+
+func (storage *PublishedStorage) internalFilelist(prefix string, hidePlusWorkaround bool) (paths []string, md5s []string, err error) {
+	paths = make([]string, 0, 1024)
+	md5s = make([]string, 0, 1024)
 	marker := ""
 	prefix = filepath.Join(storage.prefix, prefix)
 	if prefix != "" {
@@ -214,16 +267,23 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 	for {
 		contents, err := storage.bucket.List(prefix, "", marker, 1000)
 		if err != nil {
-			return nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
+			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
 		}
 		lastKey := ""
 		for _, key := range contents.Contents {
-			if prefix == "" {
-				result = append(result, key.Key)
-			} else {
-				result = append(result, key.Key[len(prefix):])
-			}
 			lastKey = key.Key
+			if storage.plusWorkaround && hidePlusWorkaround && strings.Index(lastKey, " ") != -1 {
+				// if we use plusWorkaround, we want to hide those duplicates
+				/// from listing
+				continue
+			}
+
+			if prefix == "" {
+				paths = append(paths, key.Key)
+			} else {
+				paths = append(paths, key.Key[len(prefix):])
+			}
+			md5s = append(md5s, strings.Replace(key.ETag, "\"", "", -1))
 		}
 		if contents.IsTruncated {
 			marker = contents.NextMarker
@@ -239,7 +299,7 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 		}
 	}
 
-	return result, nil
+	return paths, md5s, nil
 }
 
 // RenameFile renames (moves) file

s3/public_test.go

@@ -25,10 +25,10 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 	c.Assert(s.srv, NotNil)
 
 	auth, _ := aws.GetAuth("aa", "bb")
-	s.storage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "", "", "", false)
+	s.storage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "", "", "", false, true)
 	c.Assert(err, IsNil)
 
-	s.prefixedStorage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "lala", "", "", false)
+	s.prefixedStorage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "lala", "", "", false, true)
 	c.Assert(err, IsNil)
 
 	err = s.storage.s3.Bucket("test").PutBucket("private")
@@ -40,7 +40,7 @@ func (s *PublishedStorageSuite) TearDownTest(c *C) {
 }
 
 func (s *PublishedStorageSuite) TestNewPublishedStorage(c *C) {
-	stor, err := NewPublishedStorage("aa", "bbb", "", "", "", "", "", "", false)
+	stor, err := NewPublishedStorage("aa", "bbb", "", "", "", "", "", "", "", false, false)
 	c.Check(stor, IsNil)
 	c.Check(err, ErrorMatches, "unknown region: .*")
 }
@@ -108,6 +108,33 @@ func (s *PublishedStorageSuite) TestFilelist(c *C) {
 	c.Check(list, DeepEquals, []string{"a", "b", "c"})
 }
 
+func (s *PublishedStorageSuite) TestFilelistPlusWorkaround(c *C) {
+	s.storage.plusWorkaround = true
+	s.prefixedStorage.plusWorkaround = true
+
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")
+		c.Check(err, IsNil)
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a+b", "lala/c", "test/a+1", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a+1"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a+b", "c"})
+}
+
 func (s *PublishedStorageSuite) TestRemove(c *C) {
 	err := s.storage.bucket.Put("a/b", []byte("test"), "binary/octet-stream", "private")
 	c.Check(err, IsNil)
@@ -119,9 +146,50 @@ func (s *PublishedStorageSuite) TestRemove(c *C) {
 	c.Check(err, ErrorMatches, "The specified key does not exist.")
 }
 
-func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
-	c.Skip("multiple-delete not available in s3test")
+func (s *PublishedStorageSuite) TestRemovePlusWorkaround(c *C) {
+	s.storage.plusWorkaround = true
 
+	err := s.storage.bucket.Put("a/b+c", []byte("test"), "binary/octet-stream", "private")
+	c.Check(err, IsNil)
+
+	err = s.storage.bucket.Put("a/b", []byte("test"), "binary/octet-stream", "private")
+	c.Check(err, IsNil)
+
+	err = s.storage.Remove("a/b+c")
+	c.Check(err, IsNil)
+
+	_, err = s.storage.bucket.Get("a/b+c")
+	c.Check(err, ErrorMatches, "The specified key does not exist.")
+
+	_, err = s.storage.bucket.Get("a/b c")
+	c.Check(err, ErrorMatches, "The specified key does not exist.")
+
+	err = s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	_, err = s.storage.bucket.Get("a/b")
+	c.Check(err, ErrorMatches, "The specified key does not exist.")
+
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
+	s.storage.plusWorkaround = true
+
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")
+		c.Check(err, IsNil)
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a+b", "lala/c", "testa"})
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirsPlusWorkaround(c *C) {
 	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
 	for _, path := range paths {
 		err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")
@@ -133,8 +201,7 @@ func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
 
 	list, err := s.storage.Filelist("")
 	c.Check(err, IsNil)
-	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
-
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "testa"})
 }
 
 func (s *PublishedStorageSuite) TestRenameFile(c *C) {

s3/retry.go

@@ -0,0 +1,121 @@
+package s3
+
+// This was taken from github.com/mitchellh/goamz/amz/client.go:
+
+import (
+	"math"
+	"net"
+	"net/http"
+	"time"
+)
+
+type RetryableFunc func(*http.Request, *http.Response, error) bool
+type WaitFunc func(try int)
+type DeadlineFunc func() time.Time
+
+type ResilientTransport struct {
+	// Timeout is the maximum amount of time a dial will wait for
+	// a connect to complete.
+	//
+	// The default is no timeout.
+	//
+	// With or without a timeout, the operating system may impose
+	// its own earlier timeout. For instance, TCP timeouts are
+	// often around 3 minutes.
+	DialTimeout time.Duration
+
+	// MaxTries, if non-zero, specifies the number of times we will retry on
+	// failure. Retries are only attempted for temporary network errors or known
+	// safe failures.
+	MaxTries    int
+	ShouldRetry RetryableFunc
+	Wait        WaitFunc
+	transport   *http.Transport
+}
+
+// Convenience method for creating an http client
+func NewClient(rt *ResilientTransport) *http.Client {
+	rt.transport = &http.Transport{
+		Dial: func(netw, addr string) (net.Conn, error) {
+			c, err := net.DialTimeout(netw, addr, rt.DialTimeout)
+			if err != nil {
+				return nil, err
+			}
+			return c, nil
+		},
+		DisableKeepAlives: true,
+		Proxy:             http.ProxyFromEnvironment,
+	}
+	// TODO: Would be nice is ResilientTransport allowed clients to initialize
+	// with http.Transport attributes.
+	return &http.Client{
+		Transport: rt,
+	}
+}
+
+var retryingTransport = &ResilientTransport{
+	DialTimeout: 15 * time.Second,
+	MaxTries:    3,
+	ShouldRetry: awsRetry,
+	Wait:        ExpBackoff,
+}
+
+// Exported default client
+var RetryingClient = NewClient(retryingTransport)
+
+func (t *ResilientTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	return t.tries(req)
+}
+
+// Retry a request a maximum of t.MaxTries times.
+// We'll only retry if the proper criteria are met.
+// If a wait function is specified, wait that amount of time
+// In between requests.
+func (t *ResilientTransport) tries(req *http.Request) (res *http.Response, err error) {
+	for try := 0; try < t.MaxTries; try += 1 {
+		res, err = t.transport.RoundTrip(req)
+
+		if !t.ShouldRetry(req, res, err) {
+			break
+		}
+		if try == (t.MaxTries - 1) {
+			break
+		}
+		if res != nil {
+			res.Body.Close()
+		}
+		if t.Wait != nil {
+			t.Wait(try)
+		}
+	}
+
+	return
+}
+
+func ExpBackoff(try int) {
+	time.Sleep(100 * time.Millisecond *
+		time.Duration(math.Exp2(float64(try))))
+}
+
+// Decide if we should retry a request.
+// In general, the criteria for retrying a request is described here
+// http://docs.aws.amazon.com/general/latest/gr/api-retries.html
+func awsRetry(req *http.Request, res *http.Response, err error) bool {
+	retry := false
+
+	// Retry if there's a temporary network error.
+	if neterr, ok := err.(net.Error); ok {
+		if neterr.Temporary() {
+			retry = true
+		}
+	}
+
+	// Retry if we get a 5xx series error.
+	if res != nil {
+		if res.StatusCode >= 500 && res.StatusCode < 600 {
+			retry = true
+		}
+	}
+
+	return retry
+}

swift/public_test.go

@@ -1,21 +1,21 @@
 package swift
 
 import (
-	"github.com/ncw/swift/swifttest"
-	"github.com/smira/aptly/files"
-
+	"fmt"
 	. "gopkg.in/check.v1"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"path/filepath"
-)
+	"time"
 
-const (
-	TestAddress = "localhost:5324"
-	AuthURL     = "http://" + TestAddress + "/v1.0"
+	"github.com/ncw/swift/swifttest"
+
+	"github.com/smira/aptly/files"
 )
 
 type PublishedStorageSuite struct {
+	TestAddress, AuthURL     string
 	srv                      *swifttest.SwiftServer
 	storage, prefixedStorage *PublishedStorage
 }
@@ -24,14 +24,20 @@ var _ = Suite(&PublishedStorageSuite{})
 
 func (s *PublishedStorageSuite) SetUpTest(c *C) {
 	var err error
-	s.srv, err = swifttest.NewSwiftServer(TestAddress)
+
+	rand.Seed(int64(time.Now().Nanosecond()))
+
+	s.TestAddress = fmt.Sprintf("localhost:%d", rand.Intn(10000)+20000)
+	s.AuthURL = "http://" + s.TestAddress + "/v1.0"
+
+	s.srv, err = swifttest.NewSwiftServer(s.TestAddress)
 	c.Assert(err, IsNil)
 	c.Assert(s.srv, NotNil)
 
-	s.storage, err = NewPublishedStorage("swifttest", "swifttest", AuthURL, "", "", "test", "")
+	s.storage, err = NewPublishedStorage("swifttest", "swifttest", s.AuthURL, "", "", "test", "")
 	c.Assert(err, IsNil)
 
-	s.prefixedStorage, err = NewPublishedStorage("swifttest", "swifttest", AuthURL, "", "", "test", "lala")
+	s.prefixedStorage, err = NewPublishedStorage("swifttest", "swifttest", s.AuthURL, "", "", "test", "lala")
 	c.Assert(err, IsNil)
 
 	s.storage.conn.ContainerCreate("test", nil)
@@ -42,7 +48,7 @@ func (s *PublishedStorageSuite) TearDownTest(c *C) {
 }
 
 func (s *PublishedStorageSuite) TestNewPublishedStorage(c *C) {
-	stor, err := NewPublishedStorage("swifttest", "swifttest", AuthURL, "", "", "", "")
+	stor, err := NewPublishedStorage("swifttest", "swifttest", s.AuthURL, "", "", "", "")
 	c.Check(stor, NotNil)
 	c.Check(err, IsNil)
 }

system/changes/hardlink_0.2.1.dsc

@@ -0,0 +1,30 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+Format: 1.0
+Source: hardlink
+Binary: hardlink
+Architecture: any
+Version: 0.2.1
+Maintainer: Julian Andres Klode <jak@debian.org>
+Homepage: http://jak-linux.org/projects/hardlink/
+Standards-Version: 3.9.3
+Vcs-Browser: http://git.debian.org/?p=users/jak/hardlink.git;a=summary
+Vcs-Git: git://git.debian.org/git/users/jak/hardlink.git
+Build-Depends: debhelper (>= 9), pkg-config, libpcre3-dev
+Package-List: 
+ hardlink deb utils optional
+Checksums-Sha1: 
+ 6e95b8cba450343ab4dc01902e521f29fbd87ac2 12516 hardlink_0.2.1.tar.gz
+Checksums-Sha256: 
+ 4df0adce005526a1f0e1b38171ddb1f017faae9205f5b1c6dfb0fb4207767271 12516 hardlink_0.2.1.tar.gz
+Files: 
+ 8e2caa4d82f228bac08dc9a38bc6edb3 12516 hardlink_0.2.1.tar.gz
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.12 (GNU/Linux)
+
+iEYEARECAAYFAlUFwywACgkQIdu4nBbbPm2M5wCg0pHD8adE1rY1/DpZ4efRuMXY
+MPMAni4xUtyAnwIvkk3MCE2rFrGP3L78
+=5CU1
+-----END PGP SIGNATURE-----

system/changes/hardlink_0.2.1_amd64.changes

@@ -0,0 +1,39 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+Format: 1.8
+Date: Sat, 12 May 2014 12:57:02 +0200
+Source: hardlink
+Binary: hardlink
+Architecture: source amd64
+Version: 0.2.1
+Distribution: unstable
+Urgency: low
+Maintainer: Julian Andres Klode <jak@debian.org>
+Changed-By: Aptly Tester (don't use it) <test@aptly.info>
+Description: 
+ hardlink   - Hardlinks multiple copies of the same file
+Changes: 
+ hardlink (0.2.1) unstable; urgency=low
+ .
+   * Update just to try it out :)
+Checksums-Sha1: 
+ ff306b8f923653b78e00c45ebbc6c1c734859cdf 949 hardlink_0.2.1.dsc
+ 6e95b8cba450343ab4dc01902e521f29fbd87ac2 12516 hardlink_0.2.1.tar.gz
+ 1ac0e962854dff46f14fa7943746660d3cad1679 12468 hardlink_0.2.1_amd64.deb
+Checksums-Sha256: 
+ c0d7458aa2ca3886cd6885f395a289efbc9a396e6765cbbca45f51fde859ea70 949 hardlink_0.2.1.dsc
+ 4df0adce005526a1f0e1b38171ddb1f017faae9205f5b1c6dfb0fb4207767271 12516 hardlink_0.2.1.tar.gz
+ 668399580590bf1ffcd9eb161b6e574751e15f71820c6e08245dac7c5111a0ee 12468 hardlink_0.2.1_amd64.deb
+Files: 
+ 4efce26825af5842f43961096dd890b3 949 utils optional hardlink_0.2.1.dsc
+ 8e2caa4d82f228bac08dc9a38bc6edb3 12516 utils optional hardlink_0.2.1.tar.gz
+ 2081e20b36c47f82811c25841cc0e41b 12468 utils optional hardlink_0.2.1_amd64.deb
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.12 (GNU/Linux)
+
+iEYEARECAAYFAlUFwywACgkQIdu4nBbbPm1DLACgwW4V8qLQC/QHC/7+t3Iq47Ez
+eesAn3ZYLQvLYRw3wPTKVAPI+AW6Fjxi
+=hRBo
+-----END PGP SIGNATURE-----

system/changes/uploaders1.json

@@ -0,0 +1,7 @@
+// no groups, no rules => deny all
+{
+    "groups": {
+    },
+    "rules": [
+    ]
+}

system/changes/uploaders2.json

@@ -0,0 +1,14 @@
+
+{
+    "groups": {
+        "developers": ["21DBB89C16DB3E6D", "37E1C17570096AD1"],
+    },
+    "rules": [
+        { "condition": "Source (dangerous) | Source (kernel)",
+          "deny": ["*"],
+        },
+        { "condition": "Source (hardlink)",
+          "allow": ["developers", "admins"],
+        }
+    ]
+}

system/changes/uploaders3.json

@@ -0,0 +1 @@
+{

system/changes/uploaders4.json

@@ -0,0 +1,14 @@
+
+{
+    "groups": {
+        "developers": ["21DBB89C16DB3E6D", "37E1C17570096AD1"],
+    },
+    "rules": [
+        { "condition": "Source (dangerous) | Source (kernel)",
+          "deny": ["*"],
+        },
+        { "condition": "Source (hardlink",
+          "allow": ["developers", "admins"],
+        }
+    ]
+}

system/files/flat.key

@@ -1,19 +1,19 @@
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1.4.5 (GNU/Linux)
 
-mQGiBE9p65cRBACFOL5YS4kW6xieXa98meE+RVu1hfBi1n7ajAy+ZQOfNa2Xb9if
-H8WAcwJD4cTZYB19/O/xVxCYf1hnC/T34XGC5PUzMzBDKde86UDqvT4YNHDQA/E1
-I6UUzE0MgLINO4Dt7Mw62koVrlPXklc2Zn83ucZB7YgBzJOIBFUQLghikwCgnubS
-n/9lw8Hm8CIsg4nWtHwHGPED/jXIsH7ON3PBx2wIdRsealsx5sPGHQSlq1grRHcN
-YT5/glXmVqnexY/+IFhcpjjb3vMMQ5LYq8+bDWGVMQx3GZrJs66rwPbo4kZ92OdC
-RTnY/nznJlf5gS86DaFl+NFuO7F1k8ju4CurXXGXPF7nk8cgV6CrYHz1AtNyLVqa
-306IA/9j9rdD/MY9SYT16eFMo7C2ieIS0RxxU3q9w0e8EucQKiHWMtjTPJ0Ik0GO
-TY5lAPasnD6ZBA15XSiTi2Ck2QoZQZCxdtId/nL7lNG4+vQ8HACNDkxxK4yHJiFa
-frMdlWi5cYgAMYzbYPekbhaamDR7Gh4NU7z72QZTPELKyZD/pbRGaG9tZTpEZWVw
-RGl2ZXIxOTc1IE9CUyBQcm9qZWN0IDxob21lOkRlZXBEaXZlcjE5NzVAYnVpbGQu
-b3BlbnN1c2Uub3JnPohlBBMRAgAmBQJPaeuXAhsDBQkEHrAABgsJCAcDAgQVAggD
-BBYCAwECHgECF4AACgkQuCWWGBBIwf+tEgCcDEzTAilZDvOr0OKHHmguuKFXoHMA
-ljX7B6nKOYoiHoGpBeOwr8U5ZB6IRgQTEQIABgUCT2nrlwAKCRA7MBG3a51lI7Rt
-AJ4mYeomQiHHfd+7c8T0JhbGKUIDlACglHyTlouU5vCpUEHDyLvwrHFylpk=
-=b/6f
+mQGiBEeWWZoRBADvY6hFi/SwuwZRLbT2yrW/l+/0zsNXusK/s2K6jjA3i5oDWwG+
+wky3ef+9au2qaxd2spt8iq0x5ph8aI/1YEkdFbHe/dfItcc7NAWl2RTworogkz+R
+4leJNU3tQJNJQYml33LQyNhgQsmMvmZ6xP4NxJZIrWT9nZxiGexb24uTGwCgkOWV
+CpqucQTHnow1ok75gVMRHdkD/3EOtaBAkj5gECBc2WnXk+UzCYBjV1QXmBaLy4sq
+vIq0b3ja+zBJzc2mAviWgNuo2h56dYEsBLaQjkEdJKUEGdfiRkDZQjZEPMrT1HZD
+L2oYzR4xlwSTqR3dtBzXbLhIvfRUNw1CuNVQG8I+r7aBEje7JAVH8X4poGV3oCBh
+faXDA/0a/01gn1d58uSLwDKmBQpUngX/K1U2PchUnJd88DSRqIpzX1Cwk2iu9lJS
+ukE9FV2FNb4heKS2HRXyEz/kEONbSC4G1jSzl0keKrc8RwaX2uTolDGqpyOkXSd+
+WHUcnbzQ5MLahLB3QOocrLc0w6wwPKk2fA5F51mckMUgDtdQ2rQ8aG9tZTptb25r
+ZXlpcSBPQlMgUHJvamVjdCA8aG9tZTptb25rZXlpcUBidWlsZC5vcGVuc3VzZS5v
+cmc+iGYEExECACYFAk/y8O8CGwMFCQx7R1UGCwkIBwMCBBUCCAMEFgIDAQIeAQIX
+gAAKCRCooNd/3/r8Q3MZAJ45GODOQT+bFI8Zjq0C93L7oMxptQCgh/lNR+pYmUcT
+hb1PQ20qsfV5gJuIRgQTEQIABgUCR5ZZmgAKCRA7MBG3a51lI0jzAJsHZxWi1Db3
+J76+37rmZ/2riTo93QCfS5pFjOdqaRPjbfb6bCHLedhhHlM=
+=p7/n
 -----END PGP PUBLIC KEY BLOCK-----

system/lib.py

@@ -154,6 +154,7 @@ class BaseTest(object):
         if not hasattr(command, "__iter__"):
             params = {
                 'files': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files"),
+                'changes': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "changes"),
                 'udebs': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "udebs"),
                 'testfiles': os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), self.__class__.__name__),
                 'aptlyroot': os.path.join(os.environ["HOME"], ".aptly"),
@@ -235,6 +236,8 @@ class BaseTest(object):
             self.verify_match(self.get_gold(gold_name), contents, match_prepare=match_prepare)
         except:
             if self.captureResults:
+                if match_prepare is not None:
+                    contents = match_prepare(contents)
                 with open(self.get_gold_filename(gold_name), "w") as f:
                     f.write(contents)
             else:

system/run.py

@@ -7,6 +7,7 @@ import inspect
 import fnmatch
 import sys
 import traceback
+import random
 
 from lib import BaseTest
 from s3_lib import S3Test
@@ -98,6 +99,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
 
 if __name__ == "__main__":
     os.chdir(os.path.realpath(os.path.dirname(sys.argv[0])))
+    random.seed()
     include_long_tests = False
     capture_results = False
     tests = None

system/t01_version/VersionTest_gold

@@ -1 +1 @@
-aptly version: 0.9.5
+aptly version: 0.9.6

system/t03_help/MainHelpTest_gold

@@ -13,7 +13,7 @@ package environment to new version.
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t03_help/MainTest_gold

@@ -21,7 +21,7 @@ Use "aptly help <command>" for more information about a command.
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t03_help/MirrorCreateHelpTest_gold

@@ -15,7 +15,7 @@ Example:
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t03_help/MirrorCreateTest_gold

@@ -6,7 +6,7 @@ aptly mirror create - create new mirror
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t03_help/MirrorHelpTest_gold

@@ -17,7 +17,7 @@ Use "mirror help <command>" for more information about a command.
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t03_help/MirrorTest_gold

@@ -17,7 +17,7 @@ Use "mirror help <command>" for more information about a command.
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t03_help/WrongFlagTest_gold

@@ -7,7 +7,7 @@ aptly mirror create - create new mirror
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t04_mirror/CreateMirror11Test_mirror_show

@@ -11,10 +11,10 @@ Information from release file:
 Architectures: amd64 armel i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 sparc
 Codename: squeeze
 Components: main contrib non-free
-Date: Sat, 19 Jul 2014 11:02:06 UTC
+Date: Sat, 25 Apr 2015 11:01:14 UTC
 Description:  Debian 6.0.10 Released 19 July 2014
 
 Label: Debian
 Origin: Debian
-Suite: oldstable
+Suite: oldoldstable
 Version: 6.0.10

system/t04_mirror/CreateMirror13Test_mirror_show

@@ -11,10 +11,10 @@ Information from release file:
 Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
 Codename: wheezy
 Components: main contrib non-free
-Date: Sat, 10 Jan 2015 11:18:41 UTC
-Description:  Debian 7.8 Released 10 January 2015
+Date: Sat, 05 Sep 2015 11:44:23 UTC
+Description:  Debian 7.9 Released 05 September 2015
 
 Label: Debian
 Origin: Debian
-Suite: stable
-Version: 7.8
+Suite: oldstable
+Version: 7.9

system/t04_mirror/CreateMirror14Test_gold

@@ -1,8 +1,8 @@
-Downloading http://download.opensuse.org/repositories/home:/DeepDiver1975/xUbuntu_10.04/InRelease...
-Downloading http://download.opensuse.org/repositories/home:/DeepDiver1975/xUbuntu_10.04/Release...
-Downloading http://download.opensuse.org/repositories/home:/DeepDiver1975/xUbuntu_10.04/Release.gpg...
-gpgv: Signature made Tue May 21 23:01:30 2013 MSK using DSA key ID 1048C1FF
-gpgv: Good signature from "home:DeepDiver1975 OBS Project <home:DeepDiver1975@build.opensuse.org>"
+Downloading http://download.opensuse.org/repositories/home:/monkeyiq/Debian_7.0/InRelease...
+Downloading http://download.opensuse.org/repositories/home:/monkeyiq/Debian_7.0/Release...
+Downloading http://download.opensuse.org/repositories/home:/monkeyiq/Debian_7.0/Release.gpg...
+gpgv:  DSA key ID DFFAFC43
+gpgv: Good signature from "home:monkeyiq OBS Project <home:monkeyiq@build.opensuse.org>"
 
-Mirror [mirror14]: http://download.opensuse.org/repositories/home:/DeepDiver1975/xUbuntu_10.04/ ./ successfully added.
+Mirror [mirror14]: http://download.opensuse.org/repositories/home:/monkeyiq/Debian_7.0/ ./ successfully added.
 You can run 'aptly mirror update mirror14' to download repository contents.

system/t04_mirror/CreateMirror14Test_mirror_show

@@ -1,5 +1,5 @@
 Name: mirror14
-Archive Root URL: http://download.opensuse.org/repositories/home:/DeepDiver1975/xUbuntu_10.04/
+Archive Root URL: http://download.opensuse.org/repositories/home:/monkeyiq/Debian_7.0/
 Distribution: ./
 Components: 
 Architectures: 
@@ -8,9 +8,11 @@ Download .udebs: no
 Last update: never
 
 Information from release file:
-Date: Tue May 21 21:01:30 2013
-Description:  Open Build Service home:DeepDiver1975 xUbuntu_10.04
+Architectures: i386 amd64
+Archive: Debian_7.0
+Codename: Debian_7.0
+Date: Fri Apr 25 04:32:23 2014
+Description:  monkeyiq's Home Project (Debian_7.0)
 
-Label: DeepDiver1975's Home Project (xUbuntu_10.04)
-Origin: Open Build Service home:DeepDiver1975 xUbuntu_10.04
-Version: 0.00
+Label: home:monkeyiq
+Origin: obs://build.opensuse.org/home:monkeyiq/Debian_7.0

system/t04_mirror/CreateMirror17Test_mirror_show

@@ -11,10 +11,10 @@ Information from release file:
 Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
 Codename: wheezy
 Components: main contrib non-free
-Date: Sat, 10 Jan 2015 11:18:41 UTC
-Description:  Debian 7.8 Released 10 January 2015
+Date: Sat, 05 Sep 2015 11:44:23 UTC
+Description:  Debian 7.9 Released 05 September 2015
 
 Label: Debian
 Origin: Debian
-Suite: stable
-Version: 7.8
+Suite: oldstable
+Version: 7.9

system/t04_mirror/CreateMirror19Test_gold

@@ -1,4 +1,6 @@
 Downloading http://security.debian.org/dists/wheezy/updates/InRelease...
+gpgv:  RSA key ID C857C906
+gpgv: Good signature from "Debian Security Archive Automatic Signing Key (8/jessie) <ftpmaster@debian.org>"
 gpgv:  RSA key ID 46925553
 gpgv: Good signature from "Debian Archive Automatic Signing Key (7.0/wheezy) <ftpmaster@debian.org>"
 

system/t04_mirror/CreateMirror19Test_mirror_show

@@ -15,5 +15,5 @@ Description:  Debian 7.0 Security Updates
 
 Label: Debian-Security
 Origin: Debian
-Suite: stable
+Suite: oldstable
 Version: 7.0

system/t04_mirror/CreateMirror1Test_mirror_show

@@ -11,10 +11,10 @@ Information from release file:
 Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
 Codename: wheezy
 Components: main contrib non-free
-Date: Sat, 10 Jan 2015 11:18:41 UTC
-Description:  Debian 7.8 Released 10 January 2015
+Date: Sat, 05 Sep 2015 11:44:23 UTC
+Description:  Debian 7.9 Released 05 September 2015
 
 Label: Debian
 Origin: Debian
-Suite: stable
-Version: 7.8
+Suite: oldstable
+Version: 7.9

system/t04_mirror/CreateMirror22Test_mirror_show

@@ -17,5 +17,5 @@ Description:  Debian 7.0 Security Updates
 
 Label: Debian-Security
 Origin: Debian
-Suite: stable
+Suite: oldstable
 Version: 7.0

system/t04_mirror/CreateMirror24Test_gold

@@ -1,4 +1,6 @@
 Downloading http://security.debian.org/dists/wheezy/updates/InRelease...
+gpgv:  RSA key ID C857C906
+gpgv: Good signature from "Debian Security Archive Automatic Signing Key (8/jessie) <ftpmaster@debian.org>"
 gpgv:  RSA key ID 46925553
 gpgv: Good signature from "Debian Archive Automatic Signing Key (7.0/wheezy) <ftpmaster@debian.org>"
 

system/t04_mirror/CreateMirror25Test_mirror_show

@@ -11,10 +11,10 @@ Information from release file:
 Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
 Codename: wheezy
 Components: main contrib non-free
-Date: Sat, 10 Jan 2015 11:18:41 UTC
-Description:  Debian 7.8 Released 10 January 2015
+Date: Sat, 05 Sep 2015 11:44:23 UTC
+Description:  Debian 7.9 Released 05 September 2015
 
 Label: Debian
 Origin: Debian
-Suite: stable
-Version: 7.8
+Suite: oldstable
+Version: 7.9

system/t04_mirror/CreateMirror27Test_mirror_show

@@ -10,8 +10,8 @@ Last update: never
 Information from release file:
 Architectures: amd64 i386 source
 Codename: wheezy
-Components: openmanage openmanage/801 openmanage/740 openmanage/730
-Date: Wed, 22 Oct 2014 18:50:39 UTC
+Components: openmanage openmanage/820 openmanage/810 openmanage/801 openmanage/740 openmanage/730
+Date: Mon, 14 Sep 2015 21:24:43 UTC
 Description:  Unofficial Dell OMSA build for Ubuntu
 
 Label: Dell OMSA Archive

system/t04_mirror/CreateMirror28Test_mirror_show

@@ -10,8 +10,7 @@ Last update: never
 Information from release file:
 Architectures: i386 amd64
 Codename: dist
-Components: mongodb
-Date: Wed, 25 Feb 2015 17:35:02 UTC
+Components: 10gen
 Description:  mongodb packages
 
 Label: mongodb

system/t04_mirror/CreateMirror2Test_mirror_show

@@ -11,10 +11,10 @@ Information from release file:
 Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
 Codename: wheezy
 Components: main contrib non-free
-Date: Sat, 10 Jan 2015 11:18:41 UTC
-Description:  Debian 7.8 Released 10 January 2015
+Date: Sat, 05 Sep 2015 11:44:23 UTC
+Description:  Debian 7.9 Released 05 September 2015
 
 Label: Debian
 Origin: Debian
-Suite: stable
-Version: 7.8
+Suite: oldstable
+Version: 7.9

system/t04_mirror/CreateMirror3Test_mirror_show

@@ -11,10 +11,10 @@ Information from release file:
 Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
 Codename: wheezy
 Components: main contrib non-free
-Date: Sat, 10 Jan 2015 11:18:41 UTC
-Description:  Debian 7.8 Released 10 January 2015
+Date: Sat, 05 Sep 2015 11:44:23 UTC
+Description:  Debian 7.9 Released 05 September 2015
 
 Label: Debian
 Origin: Debian
-Suite: stable
-Version: 7.8
+Suite: oldstable
+Version: 7.9