Release 0.9.7.

Add link to scala sbt plugin

.goxc.json

@@ -0,0 +1,38 @@
+{
+	"AppName": "aptly",
+	"ArtifactsDest": "xc-out/",
+	"TasksExclude": [
+		"rmbin"
+	],
+    "TasksAppend": [
+        "bintray"
+    ],
+    "TaskSettings": {
+        "deb": {
+            "metadata": {
+                "maintainer": "Andrey Smirnov",
+                "maintainerEmail": "me@smira.ru",
+                "description": "Debian repository management tool"
+            },
+            "metadata-deb": {
+                "License": "MIT",
+                "Homepage": "https://www.aptly.info/",
+                "Recommends": "bzip2, graphviz, xz-utils",
+                "Depends": ""
+            },
+            "other-mapped-files": {
+                "/": "root/"
+            }
+        },
+        "bintray": {
+            "repository": "aptly",
+            "subject": "smira",
+            "package": "aptly",
+            "downloadspage": "bintray.md"
+        }
+    },
+	"Arch": "386 amd64",
+	"Os": "linux darwin freebsd",
+	"MainDirsExclude": "man,_vendor",
+	"ConfigVersion": "0.9"
+}

.travis.yml

@@ -1,31 +1,36 @@
+sudo: false
+
 language: go
 
 go:
-  - 1.3.3
+  - 1.4
+  - 1.5
   - tip
 
+addons:
+  apt:
+    packages:
+    - python-virtualenv
+    - graphviz
+
 env:
   global:
     - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
     - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
     - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
 before_install:
-  - sudo apt-get update -qq
-  - sudo apt-get install -y python-virtualenv graphviz
   - virtualenv env
   - . env/bin/activate
   - pip install boto requests python-swiftclient
 install:
   - make prepare
 
-
 script: make travis
 
 matrix:
     allow_failures:
         - go: tip
 
-
 notifications:
   webhooks:
     urls:

AUTHORS

@@ -15,3 +15,10 @@ List of contributors, in chronological order:
 * Michael Koval (https://github.com/mkoval)
 * Alexander Guy (https://github.com/alexanderguy)
 * Sebastien Badia (https://github.com/sbadia)
+* Szymon Sobik (https://github.com/sobczyk)
+* Paul Krohn (https://github.com/paul-krohn)
+* Vincent Bernat (https://github.com/vincentbernat)
+* x539 (https://github.com/x539)
+* Phil Frost (https://github.com/bitglue)
+* Benoit Foucher (https://github.com/bentoi)
+* Geoffrey Thomas (https://github.com/geofft)

Gomfile

@@ -1,21 +1,26 @@
-gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
-gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2'
-gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556'
-gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
 gom 'github.com/AlekSi/pointer', :commit => '5f6d527dae3d678b46fbb20331ddf44e2b841943'
+gom 'github.com/awalterschulze/gographviz', :commit => '20d1f693416d9be045340150094aa42035a41c9e'
+gom 'github.com/aws/aws-sdk-go', :commit => 'a170e9cb76475a0da7c0326a13cc2b39e9244b3b'
 gom 'github.com/cheggaaa/pb', :commit => '2c1b74620cc58a81ac152ee2d322e28c806d81ed'
+gom 'github.com/DisposaBoy/JsonConfigReader', :commit => '33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4'
 gom 'github.com/gin-gonic/gin', :commit => 'b1758d3bfa09e61ddbc1c9a627e936eec6a170de'
+gom 'github.com/go-ini/ini', :commit => 'afbd495e5aaea13597b5e14fe514ddeaa4d76fc3'
 gom 'github.com/jlaffaye/ftp', :commit => 'fec71e62e457557fbe85cefc847a048d57815d76'
+gom 'github.com/jmespath/go-jmespath', :commit => '0b12d6b521d83fc7f755e7cfc1b1fbdd35a01a74'
 gom 'github.com/julienschmidt/httprouter', :commit => '46807412fe50aaceb73bb57061c2230fd26a1640'
 gom 'github.com/mattn/go-shellwords', :commit => 'c7ca6f94add751566a61cf2199e1de78d4c3eee4'
-gom 'github.com/mitchellh/goamz/s3', :commit => 'e7664b32019f31fd1bdf33f9e85f28722f700405'
-gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
+gom 'github.com/mkrautz/goar', :commit => '282caa8bd9daba480b51f1d5a988714913b97aad'
+gom 'github.com/mxk/go-flowrate/flowrate', :commit => 'cca7078d478f8520f85629ad7c68962d31ed7682'
 gom 'github.com/ncw/swift', :commit => '384ef27c70645e285f8bb9d02276bf654d06027e'
+gom 'github.com/smira/go-aws-auth', :commit => '0070896e9d7f4f9f2d558532b2d896ce2239992a'
+gom 'github.com/smira/go-xz', :commit => '0c531f070014e218b21f3cfca801cc992d52726d'
 gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
 gom 'github.com/smira/flag', :commit => '357ed3e599ffcbd4aeaa828e1d10da2df3ea5107'
 gom 'github.com/smira/go-ftp-protocol/protocol', :commit => '066b75c2b70dca7ae10b1b88b47534a3c31ccfaa'
-gom 'github.com/syndtr/gosnappy/snappy', :commit => 'ce8acff4829e0c2458a67ead32390ac0a381c862'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => '97e257099d2ab9578151ba85e2641e2cd14d3ca8'
+gom 'github.com/smira/go-uuid/uuid', :commit => 'ed3ca8a15a931b141440a7e98e4f716eec255f7d'
+gom 'github.com/smira/lzma', :commit => '7f0af6269940baa2c938fabe73e0d7ba41205683'
+gom 'github.com/golang/snappy', :commit => '723cc1e459b8eea2dea4583200fd60757d40097a'
+gom 'github.com/syndtr/goleveldb/leveldb', :commit => '917f41c560270110ceb73c5b38be2a9127387071'
 gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
 gom 'github.com/vaughan0/go-ini', :commit => 'a98ad7ee00ec53921f08832bc06ecf7fd600e6a1'
 gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2013-2014 Andrey Smirnov. All rights reserved.
+Copyright 2013-2015 aptly authors. All rights reserved.
 
 MIT License
 

Makefile

@@ -59,17 +59,6 @@ mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-package:
-	rm -rf root/
-	mkdir -p root/usr/bin/ root/usr/share/man/man1/ root/etc/bash_completion.d
-	cp $(BINPATH)/aptly root/usr/bin
-	cp man/aptly.1 root/usr/share/man/man1
-	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
-	gzip root/usr/share/man/man1/aptly.1
-	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
-	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 --deb-recommends graphviz -C root/ .
-	mv aptly_$(VERSION)_*.deb ~
-
 src-package:
 	rm -rf aptly-$(VERSION)
 	mkdir -p aptly-$(VERSION)/src/github.com/smira/aptly/
@@ -83,4 +72,12 @@ src-package:
 	rm -rf aptly-$(VERSION)
 	curl -T aptly-$(VERSION)-src.tar.bz2 -usmira:$(BINTRAY_KEY) https://api.bintray.com/content/smira/aptly/aptly/$(VERSION)/$(VERSION)/aptly-$(VERSION)-src.tar.bz2
 
+goxc:
+	rm -rf root/
+	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d
+	cp man/aptly.1 root/usr/share/man/man1
+	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
+	gzip root/usr/share/man/man1/aptly.1
+	gom exec goxc -pv=$(VERSION) -max-processors=4 $(GOXC_OPTS)
+
 .PHONY: coverage.out

README.rst

@@ -48,7 +48,7 @@ To install aptly on Debian/Ubuntu, add new repository to /etc/apt/sources.list::
 
 And import key that is used to sign the release::
 
-    $ apt-key adv --keyserver keys.gnupg.net --recv-keys E083A3782A194991
+    $ apt-key adv --keyserver keys.gnupg.net --recv-keys 9E3E53F19C7DE460
 
 After that you can install aptly as any other software package::
 
@@ -64,7 +64,7 @@ If you would like to use nightly builds (unstable), please use following reposit
 
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.3+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.4+ required)::
 
     go get -u github.com/mattn/gom
     mkdir -p $GOPATH/src/github.com/smira/aptly
@@ -78,4 +78,34 @@ should work as well, but might fail or produce different result (if external lib
 
 If you don't have Go installed (or older version), you can easily install Go using `gvm <https://github.com/moovweb/gvm/>`_.
 
+Integrations
+------------
 
+Vagrant:
+
+-   `Vagrant configuration <https://github.com/sepulworld/aptly-vagrant>`_ by
+    Zane Williamson, allowing to bring two virtual servers, one with aptly installed
+    and another one set up to install packages from repository published by aptly
+
+Docker:
+
+-    `Docker container <https://github.com/mikepurvis/aptly-docker>`_ with aptly inside by Mike Purvis
+-    `Docker container <https://github.com/bryanhong/docker-aptly>`_ with aptly and nginx by Bryan Hong
+
+With configuration management systems:
+
+-   `Chef cookbook <https://github.com/hw-cookbooks/aptly>`_ by Aaron Baer
+    (Heavy Water Operations, LLC)
+-   `Puppet module <https://github.com/alphagov/puppet-aptly>`_ by
+    Government Digital Services
+-   `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by
+    Forrest Alvarez and Brian Jackson
+-   `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine
+
+CLI for aptly API:
+
+-   `Ruby aptly CLI/library <https://github.com/sepulworld/aptly_cli>`_ by Zane Williamson
+
+Scala sbt:
+
+-   `sbt aptly plugin <https://github.com/amalakar/sbt-aptly>`_ by Arup Malakar

api/api.go

@@ -22,35 +22,80 @@ func apiVersion(c *gin.Context) {
 	c.JSON(200, gin.H{"Version": aptly.Version})
 }
 
-// Periodically flushes CollectionFactory to free up memory used by collections,
-// flushing caches.
+const (
+	ACQUIREDB = iota
+	RELEASEDB
+)
+
+// Flushes all collections which cache in-memory objects
+func flushColections() {
+	// lock everything to eliminate in-progress calls
+	r := context.CollectionFactory().RemoteRepoCollection()
+	r.Lock()
+	defer r.Unlock()
+
+	l := context.CollectionFactory().LocalRepoCollection()
+	l.Lock()
+	defer l.Unlock()
+
+	s := context.CollectionFactory().SnapshotCollection()
+	s.Lock()
+	defer s.Unlock()
+
+	p := context.CollectionFactory().PublishedRepoCollection()
+	p.Lock()
+	defer p.Unlock()
+
+	// all collections locked, flush them
+	context.CollectionFactory().Flush()
+}
+
+// Periodically flushes CollectionFactory to free up memory used by
+// collections, flushing caches. If the two channels are provided,
+// they are used to acquire and release the database.
 //
 // Should be run in goroutine!
-func cacheFlusher() {
+func cacheFlusher(requests chan int, acks chan error) {
 	ticker := time.Tick(15 * time.Minute)
 
 	for {
 		<-ticker
 
-		// lock everything to eliminate in-progress calls
-		r := context.CollectionFactory().RemoteRepoCollection()
-		r.Lock()
-		defer r.Unlock()
+		// if aptly API runs in -no-lock mode,
+		// caches are flushed when DB is closed anyway, no need
+		// to flush them here
+		if requests == nil {
+			flushColections()
+		}
+	}
+}
 
-		l := context.CollectionFactory().LocalRepoCollection()
-		l.Lock()
-		defer l.Unlock()
-
-		s := context.CollectionFactory().SnapshotCollection()
-		s.Lock()
-		defer s.Unlock()
-
-		p := context.CollectionFactory().PublishedRepoCollection()
-		p.Lock()
-		defer p.Unlock()
-
-		// all collections locked, flush them
-		context.CollectionFactory().Flush()
+// Acquire database lock and release it when not needed anymore. Two
+// channels must be provided. The first one is to receive requests to
+// acquire/release the database and the second one is to send acks.
+//
+// Should be run in a goroutine!
+func acquireDatabase(requests chan int, acks chan error) {
+	clients := 0
+	for {
+		request := <-requests
+		switch request {
+		case ACQUIREDB:
+			if clients == 0 {
+				acks <- context.ReOpenDatabase()
+			} else {
+				acks <- nil
+			}
+			clients++
+		case RELEASEDB:
+			clients--
+			if clients == 0 {
+				flushColections()
+				acks <- context.CloseDatabase()
+			} else {
+				acks <- nil
+			}
+		}
 	}
 }
 
@@ -97,6 +142,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 			nil, context.DependencyOptions(), architecturesList)
 		if err != nil {
 			c.Fail(500, fmt.Errorf("unable to search: %s", err))
+			return
 		}
 	}
 

api/publish.go

@@ -97,6 +97,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		Label          string
 		Origin         string
 		ForceOverwrite bool
+		SkipContents   *bool
 		Architectures  []string
 		Signing        SigningOptions
 	}
@@ -183,6 +184,11 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 	published.Origin = b.Origin
 	published.Label = b.Label
 
+	published.SkipContents = context.Config().SkipContentsPublishing
+	if b.SkipContents != nil {
+		published.SkipContents = *b.SkipContents
+	}
+
 	duplicate := collection.CheckDuplicate(published)
 	if duplicate != nil {
 		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
@@ -199,6 +205,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 	err = collection.Add(published)
 	if err != nil {
 		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		return
 	}
 
 	c.JSON(201, published)
@@ -213,6 +220,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 	var b struct {
 		ForceOverwrite bool
 		Signing        SigningOptions
+		SkipContents   *bool
 		Snapshots      []struct {
 			Component string `binding:"required"`
 			Name      string `binding:"required"`
@@ -289,22 +297,30 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		}
 	} else {
 		c.Fail(500, fmt.Errorf("unknown published repository type"))
+		return
+	}
+
+	if b.SkipContents != nil {
+		published.SkipContents = *b.SkipContents
 	}
 
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
 	if err != nil {
 		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		return
 	}
 
 	err = collection.Update(published)
 	if err != nil {
 		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		return
 	}
 
 	err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
 		context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
 	if err != nil {
 		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		return
 	}
 
 	c.JSON(200, published)
@@ -312,6 +328,8 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 
 // DELETE /publish/:prefix/:distribution
 func apiPublishDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+
 	param := parseEscapedPath(c.Params.ByName("prefix"))
 	storage, prefix := deb.ParsePrefix(param)
 	distribution := c.Params.ByName("distribution")
@@ -326,7 +344,7 @@ func apiPublishDrop(c *gin.Context) {
 	defer collection.Unlock()
 
 	err := collection.Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress())
+		context.CollectionFactory(), context.Progress(), force)
 	if err != nil {
 		c.Fail(500, fmt.Errorf("unable to drop: %s", err))
 		return

api/repos.go

@@ -315,12 +315,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
 	}
 
-	packageFiles, failedFiles, err = deb.CollectPackageFiles(sources, reporter)
-
-	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to collect package files: %s", err))
-		return
-	}
+	packageFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
 
 	list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
 	if err != nil {
@@ -329,7 +324,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 	}
 
 	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), reporter)
+		context.CollectionFactory().PackageCollection(), reporter, nil)
 	failedFiles = append(failedFiles, failedFiles2...)
 
 	if err != nil {

api/router.go

@@ -12,11 +12,41 @@ var context *ctx.AptlyContext
 func Router(c *ctx.AptlyContext) http.Handler {
 	context = c
 
-	go cacheFlusher()
-
 	router := gin.Default()
 	router.Use(gin.ErrorLogger())
 
+	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
+		// We use a goroutine to count the number of
+		// concurrent requests. When no more requests are
+		// running, we close the database to free the lock.
+		requests := make(chan int)
+		acks := make(chan error)
+
+		go acquireDatabase(requests, acks)
+		go cacheFlusher(requests, acks)
+
+		router.Use(func(c *gin.Context) {
+			requests <- ACQUIREDB
+			err := <-acks
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+			defer func() {
+				requests <- RELEASEDB
+				err = <-acks
+				if err != nil {
+					c.Fail(500, err)
+					return
+				}
+			}()
+			c.Next()
+		})
+
+	} else {
+		go cacheFlusher(nil, nil)
+	}
+
 	root := router.Group("/api")
 
 	{

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
 // Version of aptly
-const Version = "0.9.1"
+const Version = "0.9.7"
 
 // Enable debugging features?
 const EnableDebug = false

cmd/api_serve.go

@@ -46,6 +46,7 @@ Example:
 	}
 
 	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening")
+	cmd.Flag.Bool("no-lock", false, "don't lock the database")
 
 	return cmd
 

cmd/cmd.go

@@ -2,12 +2,14 @@
 package cmd
 
 import (
+	"bytes"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"os"
+	"text/template"
 	"time"
 )
 
@@ -34,6 +36,32 @@ func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 	return
 }
 
+// PrintPackageList shows package list with specified format or default representation
+func PrintPackageList(result *deb.PackageList, format string) error {
+	if format == "" {
+		return result.ForEach(func(p *deb.Package) error {
+			context.Progress().Printf("%s\n", p)
+			return nil
+		})
+	}
+
+	formatTemplate, err := template.New("format").Parse(format)
+	if err != nil {
+		return fmt.Errorf("error parsing -format template: %s", err)
+	}
+
+	return result.ForEach(func(p *deb.Package) error {
+		b := &bytes.Buffer{}
+		err = formatTemplate.Execute(b, p.ExtendedStanza())
+		if err != nil {
+			return fmt.Errorf("error applying template: %s", err)
+		}
+		context.Progress().Printf("%s\n", b.String())
+		return nil
+	})
+
+}
+
 // LookupOption checks boolean flag with default (usually config) and command-line
 // setting
 func LookupOption(defaultValue bool, flags *flag.FlagSet, name string) (result bool) {
@@ -83,7 +111,7 @@ package environment to new version.`,
 	cmd.Flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests")
 	cmd.Flag.Bool("dep-follow-source", false, "when processing dependencies, follow from binary to Source packages")
 	cmd.Flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends")
-	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if depdency is 'a|b'")
+	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
 	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
 

cmd/db_cleanup.go

@@ -6,6 +6,7 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"sort"
+	"strings"
 )
 
 // aptly db cleanup
@@ -17,51 +18,112 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
+	verbose := context.Flags().Lookup("verbose").Value.Get().(bool)
+	dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
+
 	// collect information about references packages...
 	existingPackageRefs := deb.NewPackageRefList()
 
-	context.Progress().Printf("Loading mirrors, local repos, snapshots and published repos...\n")
+	// used only in verbose mode to report package use source
+	packageRefSources := map[string][]string{}
+
+	context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|")
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
+	}
 	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
 		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
 		if repo.RefList() != nil {
 			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("mirror %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
+
 		return nil
 	})
 	if err != nil {
 		return err
 	}
 
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
+	}
 	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
 		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
+
 		if repo.RefList() != nil {
 			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("local repo %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
+
 		return nil
 	})
 	if err != nil {
 		return err
 	}
 
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
+	}
 	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
+		}
+
 		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return err
 		}
+
 		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+
+		if verbose {
+			description := fmt.Sprintf("snapshot %s", snapshot.Name)
+			snapshot.RefList().ForEach(func(key []byte) error {
+				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+				return nil
+			})
+		}
 		return nil
 	})
 	if err != nil {
 		return err
 	}
 
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
+	}
 	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
+		}
 		if published.SourceKind != "local" {
 			return nil
 		}
@@ -72,6 +134,14 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 		for _, component := range published.Components() {
 			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+			if verbose {
+				description := fmt.Sprintf("published repository %s:%s/%s component %s",
+					published.Storage, published.Prefix, published.Distribution, component)
+				published.RefList(component).ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
 		return nil
 	})
@@ -80,38 +150,65 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	}
 
 	// ... and compare it to the list of all packages
-	context.Progress().Printf("Loading list of all packages...\n")
+	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
 	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
 
 	toDelete := allPackageRefs.Substract(existingPackageRefs)
 
 	// delete packages that are no longer referenced
-	context.Progress().Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())
 
 	// database can't err as collection factory already constructed
 	db, _ := context.Database()
-	db.StartBatch()
-	err = toDelete.ForEach(func(ref []byte) error {
-		return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
-	})
-	if err != nil {
-		return err
-	}
 
-	err = db.FinishBatch()
-	if err != nil {
-		return fmt.Errorf("unable to write to DB: %s", err)
+	if toDelete.Len() > 0 {
+		if verbose {
+			context.Progress().ColoredPrintf("@{r}List of package keys to delete:@|")
+			err = toDelete.ForEach(func(ref []byte) error {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", string(ref))
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if !dryRun {
+			db.StartBatch()
+			err = toDelete.ForEach(func(ref []byte) error {
+				return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
+			})
+			if err != nil {
+				return err
+			}
+
+			err = db.FinishBatch()
+			if err != nil {
+				return fmt.Errorf("unable to write to DB: %s", err)
+			}
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped deletion, as -dry-run has been requested.@|")
+		}
 	}
 
 	// now, build a list of files that should be present in Repository (package pool)
-	context.Progress().Printf("Building list of files referenced by packages...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())
 	context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
 
 	err = existingPackageRefs.ForEach(func(key []byte) error {
 		pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
 		if err2 != nil {
-			return err2
+			tail := ""
+			if verbose {
+				tail = fmt.Sprintf(" (sources: %s)", strings.Join(packageRefSources[string(key)], ", "))
+			}
+			if dryRun {
+				context.Progress().ColoredPrintf("@{r!}Unresolvable package reference, skipping (-dry-run): %s: %s%s",
+					string(key), err2, tail)
+				return nil
+			}
+			return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
 		}
 		paths, err2 := pkg.FilepathList(context.PackagePool())
 		if err2 != nil {
@@ -130,7 +227,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	context.Progress().ShutdownBar()
 
 	// build a list of files in the package pool
-	context.Progress().Printf("Building list of files in package pool...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files in package pool...@|")
 	existingFiles, err := context.PackagePool().FilepathList(context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to collect file paths: %s", err)
@@ -140,28 +237,43 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
 
 	// delete files that are no longer referenced
-	context.Progress().Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced files (%d)...@|", len(filesToDelete))
 
 	if len(filesToDelete) > 0 {
-		context.Progress().InitBar(int64(len(filesToDelete)), false)
-
-		var size, totalSize int64
-		for _, file := range filesToDelete {
-			size, err = context.PackagePool().Remove(file)
-			if err != nil {
-				return err
+		if verbose {
+			context.Progress().ColoredPrintf("@{r}List of files to be deleted:@|")
+			for _, file := range filesToDelete {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", file)
 			}
-
-			context.Progress().AddBar(1)
-			totalSize += size
 		}
-		context.Progress().ShutdownBar()
 
-		context.Progress().Printf("Disk space freed: %s...\n", utils.HumanBytes(totalSize))
+		if !dryRun {
+			context.Progress().InitBar(int64(len(filesToDelete)), false)
+
+			var size, totalSize int64
+			for _, file := range filesToDelete {
+				size, err = context.PackagePool().Remove(file)
+				if err != nil {
+					return err
+				}
+
+				context.Progress().AddBar(1)
+				totalSize += size
+			}
+			context.Progress().ShutdownBar()
+
+			context.Progress().ColoredPrintf("@{w!}Disk space freed: %s...@|", utils.HumanBytes(totalSize))
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped file deletion, as -dry-run has been requested.@|")
+		}
 	}
 
-	context.Progress().Printf("Compacting database...\n")
-	err = db.CompactDB()
+	if !dryRun {
+		context.Progress().ColoredPrintf("@{w!}Compacting database...@|")
+		err = db.CompactDB()
+	} else {
+		context.Progress().ColoredPrintf("@{y!}Skipped DB compaction, as -dry-run has been requested.@|")
+	}
 
 	return err
 }
@@ -181,5 +293,8 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("verbose", false, "be verbose when loading objects/removing them")
+	cmd.Flag.Bool("dry-run", false, "don't delete anything")
+
 	return cmd
 }

cmd/graph.go

@@ -4,11 +4,13 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
 )
 
 func aptlyGraph(cmd *commander.Command, args []string) error {
@@ -34,9 +36,16 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 	tempfile.Close()
 	os.Remove(tempfile.Name())
 
-	tempfilename := tempfile.Name() + ".png"
+	format := context.Flags().Lookup("format").Value.String()
+	output := context.Flags().Lookup("output").Value.String()
 
-	command := exec.Command("dot", "-Tpng", "-o"+tempfilename)
+	if filepath.Ext(output) != "" {
+		format = filepath.Ext(output)[1:]
+	}
+
+	tempfilename := tempfile.Name() + "." + format
+
+	command := exec.Command("dot", "-T"+format, "-o"+tempfilename)
 	command.Stderr = os.Stderr
 
 	stdin, err := command.StdinPipe()
@@ -64,10 +73,18 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	err = exec.Command("open", tempfilename).Run()
-	if err != nil {
-		fmt.Printf("Rendered to PNG file: %s\n", tempfilename)
-		err = nil
+	if output != "" {
+		err = utils.CopyFile(tempfilename, output)
+		if err != nil {
+			return fmt.Errorf("unable to copy %s -> %s: %s", tempfilename, output, err)
+		}
+		_ = os.Remove(tempfilename)
+
+		fmt.Printf("Output saved to %s\n", output)
+	} else {
+		fmt.Printf("Rendered to %s file: %s, trying to open it...\n", format, tempfilename)
+
+		_ = exec.Command("open", tempfilename).Run()
 	}
 
 	return err
@@ -89,5 +106,8 @@ Example:
 `,
 	}
 
+	cmd.Flag.String("format", "png", "render graph to specified format (png, svg, pdf, etc.)")
+	cmd.Flag.String("output", "", "specify output filename, default is to open result in viewer")
+
 	return cmd
 }

cmd/mirror_search.go

@@ -21,6 +21,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/package_search.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -25,10 +24,8 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("no results")
 	}
 
-	result.ForEach(func(p *deb.Package) error {
-		context.Progress().Printf("%s\n", p)
-		return nil
-	})
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format)
 
 	return err
 }
@@ -48,5 +45,7 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-package-search", flag.ExitOnError),
 	}
 
+	cmd.Flag.String("format", "", "custom format for result printing")
+
 	return cmd
 }

cmd/publish_drop.go

@@ -23,7 +23,7 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 	storage, prefix := deb.ParsePrefix(param)
 
 	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress())
+		context.CollectionFactory(), context.Progress(), context.Flags().Lookup("force-drop").Value.Get().(bool))
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -48,5 +48,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
+
 	return cmd
 }

cmd/publish_repo.go

@@ -41,6 +41,7 @@ Example:
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")

cmd/publish_snapshot.go

@@ -116,8 +116,14 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
-	published.Origin = cmd.Flag.Lookup("origin").Value.String()
-	published.Label = cmd.Flag.Lookup("label").Value.String()
+	published.Origin = context.Flags().Lookup("origin").Value.String()
+	published.Label = context.Flags().Lookup("label").Value.String()
+
+	published.SkipContents = context.Config().SkipContentsPublishing
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
 
 	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
@@ -203,6 +209,7 @@ Example:
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")

cmd/publish_switch.go

@@ -90,6 +90,10 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 			"the same package pool.\n")
 	}
 
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
@@ -143,6 +147,7 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 

cmd/publish_update.go

@@ -54,6 +54,10 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 			"the same package pool.\n")
 	}
 
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
@@ -102,6 +106,7 @@ Example:
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd

cmd/repo.go

@@ -21,6 +21,7 @@ func makeCmdRepo() *commander.Command {
 			makeCmdRepoShow(),
 			makeCmdRepoRename(),
 			makeCmdRepoSearch(),
+			makeCmdRepoInclude(),
 		},
 	}
 }

cmd/repo_add.go

@@ -42,15 +42,12 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	var packageFiles, failedFiles []string
 
-	packageFiles, failedFiles, err = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
-	if err != nil {
-		return fmt.Errorf("unable to collect package files: %s", err)
-	}
+	packageFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
 
 	var processedFiles, failedFiles2 []string
 
 	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()})
+		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil)
 	failedFiles = append(failedFiles, failedFiles2...)
 	if err != nil {
 		return fmt.Errorf("unable to import package files: %s", err)

cmd/repo_create.go

@@ -18,6 +18,14 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
 	repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
 
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		repo.Uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+	}
+
 	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add local repo: %s", err)
@@ -47,6 +55,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "main", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_edit.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/AlekSi/pointer"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -23,16 +25,30 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	if context.Flags().Lookup("comment").Value.String() != "" {
-		repo.Comment = context.Flags().Lookup("comment").Value.String()
-	}
+	var uploadersFile *string
 
-	if context.Flags().Lookup("distribution").Value.String() != "" {
-		repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
-	}
+	context.Flags().Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "comment":
+			repo.Comment = flag.Value.String()
+		case "distribution":
+			repo.DefaultDistribution = flag.Value.String()
+		case "component":
+			repo.DefaultComponent = flag.Value.String()
+		case "uploaders-file":
+			uploadersFile = pointer.ToString(flag.Value.String())
+		}
+	})
 
-	if context.Flags().Lookup("component").Value.String() != "" {
-		repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
+	if uploadersFile != nil {
+		if *uploadersFile != "" {
+			repo.Uploaders, err = deb.NewUploadersFromFile(*uploadersFile)
+			if err != nil {
+				return err
+			}
+		} else {
+			repo.Uploaders = nil
+		}
 	}
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
@@ -63,6 +79,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_include.go

@@ -0,0 +1,234 @@
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"os"
+	"path/filepath"
+	"text/template"
+)
+
+func aptlyRepoInclude(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	verifier, err := getVerifier(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+	}
+
+	if verifier == nil {
+		verifier = &utils.GpgVerifier{}
+	}
+
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
+	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
+	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
+
+	repoTemplate, err := template.New("repo").Parse(context.Flags().Lookup("repo").Value.Get().(string))
+	if err != nil {
+		return fmt.Errorf("error parsing -repo template: %s", err)
+	}
+
+	uploaders := (*deb.Uploaders)(nil)
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+
+		for i := range uploaders.Rules {
+			uploaders.Rules[i].CompiledCondition, err = query.Parse(uploaders.Rules[i].Condition)
+			if err != nil {
+				return fmt.Errorf("error parsing query %s: %s", uploaders.Rules[i].Condition, err)
+			}
+		}
+	}
+
+	reporter := &aptly.ConsoleResultReporter{Progress: context.Progress()}
+
+	var changesFiles, failedFiles, processedFiles []string
+
+	changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)
+
+	for _, path := range changesFiles {
+		var changes *deb.Changes
+
+		changes, err = deb.NewChanges(path)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", path, err)
+			continue
+		}
+
+		err = changes.VerifyAndParse(acceptUnsigned, ignoreSignatures, verifier)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		err = changes.Prepare()
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		repoName := &bytes.Buffer{}
+		err = repoTemplate.Execute(repoName, changes.Stanza)
+		if err != nil {
+			return fmt.Errorf("error applying template to repo: %s", err)
+		}
+
+		context.Progress().Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
+
+		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(repoName.String())
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		currentUploaders := uploaders
+		if repo.Uploaders != nil {
+			currentUploaders = repo.Uploaders
+			for i := range currentUploaders.Rules {
+				currentUploaders.Rules[i].CompiledCondition, err = query.Parse(currentUploaders.Rules[i].Condition)
+				if err != nil {
+					return fmt.Errorf("error parsing query %s: %s", currentUploaders.Rules[i].Condition, err)
+				}
+			}
+		}
+
+		if currentUploaders != nil {
+			if err = currentUploaders.IsAllowed(changes); err != nil {
+				failedFiles = append(failedFiles, path)
+				reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
+					changes.ChangesName, changes.SignatureKeys, err)
+				changes.Cleanup()
+				continue
+			}
+		}
+
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to load repo: %s", err)
+		}
+
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to load packages: %s", err)
+		}
+
+		packageFiles, _ := deb.CollectPackageFiles([]string{changes.TempDir}, reporter)
+
+		var restriction deb.PackageQuery
+
+		restriction, err = changes.PackageQuery()
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		var processedFiles2, failedFiles2 []string
+
+		processedFiles2, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+			context.CollectionFactory().PackageCollection(), reporter, restriction)
+
+		if err != nil {
+			return fmt.Errorf("unable to import package files: %s", err)
+		}
+
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+		if err != nil {
+			return fmt.Errorf("unable to save: %s", err)
+		}
+
+		err = changes.Cleanup()
+		if err != nil {
+			return err
+		}
+
+		for _, file := range failedFiles2 {
+			failedFiles = append(failedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		for _, file := range processedFiles2 {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		processedFiles = append(processedFiles, path)
+	}
+
+	if !noRemoveFiles {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err := os.Remove(file)
+			if err != nil {
+				return fmt.Errorf("unable to remove file: %s", err)
+			}
+		}
+	}
+
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
+	return err
+}
+
+func makeCmdRepoInclude() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoInclude,
+		UsageLine: "include <file.changes>|<directory> ...",
+		Short:     "add packages to local repositories based on .changes files",
+		Long: `
+Command include looks for .changes files in list of arguments or specified directories. Each
+.changes file is verified, parsed, referenced files are put into separate temporary directory
+and added into local repository. Successfully imported files are removed by default.
+
+Additionally uploads could be restricted with <uploaders.json> file. Rules in this file control
+uploads based on GPG key ID of .changes file signature and queries on .changes file fields.
+
+Example:
+
+  $ aptly repo include -repo=foo-release incoming/
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-include", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("no-remove-files", false, "don't remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
+	cmd.Flag.String("repo", "{{.Distribution}}", "which repo should files go to, defaults to Distribution field of .changes file")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of .changes file signature")
+	cmd.Flag.Bool("accept-unsigned", false, "accept unsigned .changes files")
+	cmd.Flag.String("uploaders-file", "", "path to uploaders.json file")
+
+	return cmd
+}

cmd/repo_search.go

@@ -21,6 +21,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/repo_show.go

@@ -29,6 +29,9 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Comment: %s\n", repo.Comment)
 	fmt.Printf("Default Distribution: %s\n", repo.DefaultDistribution)
 	fmt.Printf("Default Component: %s\n", repo.DefaultComponent)
+	if repo.Uploaders != nil {
+		fmt.Printf("Uploaders: %s\n", repo.Uploaders)
+	}
 	fmt.Printf("Number of packages: %d\n", repo.NumPackages())
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)

cmd/run.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	ctx "github.com/smira/aptly/context"
 	"github.com/smira/commander"
+	"os"
 )
 
 // Run runs single command starting from root cmd with args, optionally initializing context
@@ -14,7 +15,7 @@ func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode
 			if !ok {
 				panic(r)
 			}
-			fmt.Println("ERROR:", fatal.Message)
+			fmt.Fprintln(os.Stderr, "ERROR:", fatal.Message)
 			returnCode = fatal.ReturnCode
 		}
 	}()

cmd/snapshot_search.go

@@ -96,10 +96,12 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		return fmt.Errorf("unable to search: %s", err)
 	}
 
-	result.ForEach(func(p *deb.Package) error {
-		context.Progress().Printf("%s\n", p)
-		return nil
-	})
+	if result.Len() == 0 {
+		return fmt.Errorf("no results")
+	}
+
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format)
 
 	return err
 }
@@ -120,6 +122,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/snapshot_verify.go

@@ -31,25 +31,25 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 
 	packageList, err := deb.NewPackageListFromRefList(snapshots[0].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
-		fmt.Errorf("unable to load packages: %s", err)
+		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
 	sourcePackageList := deb.NewPackageList()
 	err = sourcePackageList.Append(packageList)
 	if err != nil {
-		fmt.Errorf("unable to merge sources: %s", err)
+		return fmt.Errorf("unable to merge sources: %s", err)
 	}
 
 	var pL *deb.PackageList
 	for i := 1; i < len(snapshots); i++ {
 		pL, err = deb.NewPackageListFromRefList(snapshots[i].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 		if err != nil {
-			fmt.Errorf("unable to load packages: %s", err)
+			return fmt.Errorf("unable to load packages: %s", err)
 		}
 
 		err = sourcePackageList.Append(pL)
 		if err != nil {
-			fmt.Errorf("unable to merge sources: %s", err)
+			return fmt.Errorf("unable to merge sources: %s", err)
 		}
 	}
 

console/progress.go

@@ -133,19 +133,30 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
 		p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"}
 	} else {
 		// stip color marks
-		var prev rune
+		var inColorMark, inCurly bool
 		msg = strings.Map(func(r rune) rune {
-			if prev == '@' {
-				prev = 0
-				if r == '@' {
-					return r
+			if inColorMark {
+				if inCurly {
+					if r == '}' {
+						inCurly = false
+						inColorMark = false
+						return -1
+					}
+				} else {
+					if r == '{' {
+						inCurly = true
+					} else if r == '@' {
+						return '@'
+					} else {
+						inColorMark = false
+					}
 				}
 				return -1
 			}
-			prev = r
-			if r == '@' {
-				return -1
 
+			if r == '@' {
+				inColorMark = true
+				return -1
 			}
 
 			return r

context/context.go

@@ -99,7 +99,7 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			}
 
 			if err != nil {
-				fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
+				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", configLocations[0])
 				utils.SaveConfig(configLocations[0], &utils.Config)
 			}
 		}
@@ -321,9 +321,11 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 			}
 
 			var err error
-			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
-				params.Region, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
-				params.EncryptionMethod, params.PlusWorkaround)
+			publishedStorage, err = s3.NewPublishedStorage(
+				params.AccessKeyID, params.SecretAccessKey, params.SessionToken,
+				params.Region, params.Endpoint, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
+				params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel,
+				params.ForceSigV2, params.Debug)
 			if err != nil {
 				Fatal(err)
 			}

database/leveldb.go

@@ -43,7 +43,8 @@ var (
 
 func internalOpen(path string) (*leveldb.DB, error) {
 	o := &opt.Options{
-		Filter: filter.NewBloomFilter(10),
+		Filter:                 filter.NewBloomFilter(10),
+		OpenFilesCacheCapacity: 256,
 	}
 
 	return leveldb.OpenFile(path, o)
@@ -60,7 +61,7 @@ func OpenDB(path string) (Storage, error) {
 
 // RecoverDB recovers LevelDB database from corruption
 func RecoverDB(path string) error {
-	stor, err := storage.OpenFile(path)
+	stor, err := storage.OpenFile(path, false)
 	if err != nil {
 		return err
 	}

deb/changes.go

@@ -0,0 +1,273 @@
+package deb
+
+import (
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// Changes is a result of .changes file parsing
+type Changes struct {
+	Changes               string
+	Distribution          string
+	Files                 PackageFiles
+	BasePath, ChangesName string
+	TempDir               string
+	Source                string
+	Binary                []string
+	Architectures         []string
+	Stanza                Stanza
+	SignatureKeys         []utils.GpgKey
+}
+
+// NewChanges moves .changes file into temporary directory and creates Changes structure
+func NewChanges(path string) (*Changes, error) {
+	var err error
+
+	c := &Changes{
+		BasePath:    filepath.Dir(path),
+		ChangesName: filepath.Base(path),
+	}
+
+	c.TempDir, err = ioutil.TempDir(os.TempDir(), "aptly")
+	if err != nil {
+		return nil, err
+	}
+
+	// copy .changes file into temporary directory
+	err = utils.CopyFile(filepath.Join(c.BasePath, c.ChangesName), filepath.Join(c.TempDir, c.ChangesName))
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// VerifyAndParse does optional signature verification and parses changes files
+func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier utils.Verifier) error {
+	input, err := os.Open(filepath.Join(c.TempDir, c.ChangesName))
+	if err != nil {
+		return err
+	}
+	defer input.Close()
+
+	isClearSigned, err := verifier.IsClearSigned(input)
+	if err != nil {
+		return err
+	}
+
+	input.Seek(0, 0)
+
+	if !isClearSigned && !acceptUnsigned {
+		return fmt.Errorf(".changes file is not signed and unsigned processing hasn't been enabled")
+	}
+
+	if isClearSigned && !ignoreSignature {
+		keyInfo, err := verifier.VerifyClearsigned(input, false)
+		if err != nil {
+			return err
+		}
+		input.Seek(0, 0)
+
+		c.SignatureKeys = keyInfo.GoodKeys
+	}
+
+	var text *os.File
+
+	if isClearSigned {
+		text, err = verifier.ExtractClearsigned(input)
+		if err != nil {
+			return err
+		}
+		defer text.Close()
+	} else {
+		text = input
+	}
+
+	reader := NewControlFileReader(text)
+	c.Stanza, err = reader.ReadStanza(false)
+	if err != nil {
+		return err
+	}
+
+	c.Distribution = c.Stanza["Distribution"]
+	c.Changes = c.Stanza["Changes"]
+	c.Source = c.Stanza["Source"]
+	c.Binary = strings.Fields(c.Stanza["Binary"])
+	c.Architectures = strings.Fields(c.Stanza["Architecture"])
+
+	c.Files, err = c.Files.ParseSumFields(c.Stanza)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Prepare creates temporary directory, copies file there and verifies checksums
+func (c *Changes) Prepare() error {
+	var err error
+
+	for _, file := range c.Files {
+		if filepath.Dir(file.Filename) != "." {
+			return fmt.Errorf("file is not in the same folder as .changes file: %s", file.Filename)
+		}
+
+		file.Filename = filepath.Base(file.Filename)
+
+		err = utils.CopyFile(filepath.Join(c.BasePath, file.Filename), filepath.Join(c.TempDir, file.Filename))
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, file := range c.Files {
+		var info utils.ChecksumInfo
+
+		info, err = utils.ChecksumsForFile(filepath.Join(c.TempDir, file.Filename))
+		if err != nil {
+			return err
+		}
+
+		if info.Size != file.Checksums.Size {
+			return fmt.Errorf("size mismatch: expected %v != obtained %v", file.Checksums.Size, info.Size)
+		}
+
+		if info.MD5 != file.Checksums.MD5 {
+			return fmt.Errorf("checksum mismatch MD5: expected %v != obtained %v", file.Checksums.MD5, info.MD5)
+		}
+
+		if info.SHA1 != file.Checksums.SHA1 {
+			return fmt.Errorf("checksum mismatch SHA1: expected %v != obtained %v", file.Checksums.SHA1, info.SHA1)
+		}
+
+		if info.SHA256 != file.Checksums.SHA256 {
+			return fmt.Errorf("checksum mismatch SHA256 expected %v != obtained %v", file.Checksums.SHA256, info.SHA256)
+		}
+	}
+
+	return nil
+}
+
+// Cleanup removes all temporary files
+func (c *Changes) Cleanup() error {
+	if c.TempDir == "" {
+		return nil
+	}
+
+	return os.RemoveAll(c.TempDir)
+}
+
+// PackageQuery returns query that every package should match to be included
+func (c *Changes) PackageQuery() (PackageQuery, error) {
+	var archQuery PackageQuery = &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: ""}
+	for _, arch := range c.Architectures {
+		archQuery = &OrQuery{L: &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: arch}, R: archQuery}
+	}
+
+	// if c.Source is empty, this would never match
+	sourceQuery := &AndQuery{
+		L: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"},
+		R: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Source},
+	}
+
+	var binaryQuery PackageQuery
+	if len(c.Binary) > 0 {
+		binaryQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Binary[0]}
+		for _, binary := range c.Binary[1:] {
+			binaryQuery = &OrQuery{
+				L: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: binary},
+				R: binaryQuery,
+			}
+		}
+
+		binaryQuery = &AndQuery{
+			L: &NotQuery{Q: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"}},
+			R: binaryQuery}
+	}
+
+	var nameQuery PackageQuery
+	if binaryQuery == nil {
+		nameQuery = sourceQuery
+	} else {
+		nameQuery = &OrQuery{L: sourceQuery, R: binaryQuery}
+	}
+
+	return &AndQuery{L: archQuery, R: nameQuery}, nil
+}
+
+// GetField implements PackageLike interface
+func (c *Changes) GetField(field string) string {
+	return c.Stanza[field]
+}
+
+// MatchesDependency implements PackageLike interface
+func (c *Changes) MatchesDependency(d Dependency) bool {
+	return false
+}
+
+// MatchesArchitecture implements PackageLike interface
+func (c *Changes) MatchesArchitecture(arch string) bool {
+	return false
+}
+
+// GetName implements PackageLike interface
+func (c *Changes) GetName() string {
+	return ""
+}
+
+// GetVersion implements PackageLike interface
+func (c *Changes) GetVersion() string {
+	return ""
+
+}
+
+// GetArchitecture implements PackageLike interface
+func (c *Changes) GetArchitecture() string {
+	return ""
+}
+
+// CollectChangesFiles walks filesystem collecting all .changes files
+func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (changesFiles, failedFiles []string) {
+	for _, location := range locations {
+		info, err2 := os.Stat(location)
+		if err2 != nil {
+			reporter.Warning("Unable to process %s: %s", location, err2)
+			failedFiles = append(failedFiles, location)
+			continue
+		}
+		if info.IsDir() {
+			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
+				if err3 != nil {
+					return err3
+				}
+				if info.IsDir() {
+					return nil
+				}
+
+				if strings.HasSuffix(info.Name(), ".changes") {
+					changesFiles = append(changesFiles, path)
+				}
+
+				return nil
+			})
+
+			if err2 != nil {
+				reporter.Warning("Unable to process %s: %s", location, err2)
+				failedFiles = append(failedFiles, location)
+				continue
+			}
+		} else if strings.HasSuffix(info.Name(), ".changes") {
+			changesFiles = append(changesFiles, location)
+		}
+	}
+
+	sort.Strings(changesFiles)
+
+	return
+}

deb/changes_test.go

@@ -0,0 +1,91 @@
+package deb
+
+import (
+	. "gopkg.in/check.v1"
+	"os"
+	"path/filepath"
+)
+
+type ChangesSuite struct {
+	Dir, Path string
+}
+
+var _ = Suite(&ChangesSuite{})
+
+func (s *ChangesSuite) SetUpTest(c *C) {
+	s.Dir = c.MkDir()
+	s.Path = filepath.Join(s.Dir, "calamares.changes")
+
+	f, err := os.Create(s.Path)
+	c.Assert(err, IsNil)
+
+	f.WriteString(changesFile)
+	f.Close()
+}
+
+func (s *ChangesSuite) TestParseAndVerify(c *C) {
+	changes, err := NewChanges(s.Path)
+	c.Assert(err, IsNil)
+
+	err = changes.VerifyAndParse(true, true, &NullVerifier{})
+	c.Check(err, IsNil)
+
+	c.Check(changes.Distribution, Equals, "sid")
+	c.Check(changes.Files, HasLen, 4)
+	c.Check(changes.Files[0].Filename, Equals, "calamares_0+git20141127.99.dsc")
+	c.Check(changes.Files[0].Checksums.Size, Equals, int64(1106))
+	c.Check(changes.Files[0].Checksums.MD5, Equals, "05fd8f3ffe8f362c5ef9bad2f936a56e")
+	c.Check(changes.Files[0].Checksums.SHA1, Equals, "79f10e955dab6eb25b7f7bae18213f367a3a0396")
+	c.Check(changes.Files[0].Checksums.SHA256, Equals, "35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc")
+	c.Check(changes.BasePath, Equals, s.Dir)
+	c.Check(changes.Architectures, DeepEquals, []string{"source", "amd64"})
+	c.Check(changes.Source, Equals, "calamares")
+	c.Check(changes.Binary, DeepEquals, []string{"calamares", "calamares-dbg"})
+}
+
+func (s *ChangesSuite) TestPackageQuery(c *C) {
+	changes, err := NewChanges(s.Path)
+	c.Assert(err, IsNil)
+
+	err = changes.VerifyAndParse(true, true, &NullVerifier{})
+	c.Check(err, IsNil)
+
+	q, err := changes.PackageQuery()
+	c.Check(err, IsNil)
+
+	c.Check(q.String(), Equals,
+		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), ((Name (= calamares-dbg)) | (Name (= calamares)))))")
+}
+
+var changesFile = `Format: 1.8
+Date: Thu, 27 Nov 2014 13:24:53 +0000
+Source: calamares
+Binary: calamares calamares-dbg
+Architecture: source amd64
+Version: 0+git20141127.99
+Distribution: sid
+Urgency: medium
+Maintainer: Rohan Garg <rohan@kde.org>
+Changed-By: Rohan <rohan@kde.org>
+Description:
+ calamares  - distribution-independent installer framework
+ calamares-dbg - distribution-independent installer framework -- debug symbols
+Changes:
+ calamares (0+git20141127.99) sid; urgency=medium
+ .
+   * Update from git
+Checksums-Sha1:
+ 79f10e955dab6eb25b7f7bae18213f367a3a0396 1106 calamares_0+git20141127.99.dsc
+ 294c28e2c8e34e72ca9ee0d9da5c14f3bf4188db 2694800 calamares_0+git20141127.99.tar.xz
+ d6c26c04b5407c7511f61cb3e3de60c4a1d6c4ff 1698924 calamares_0+git20141127.99_amd64.deb
+ a3da632d193007b0d4a1aff73159fde1b532d7a8 12835902 calamares-dbg_0+git20141127.99_amd64.deb
+Checksums-Sha256:
+ 35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc 1106 calamares_0+git20141127.99.dsc
+ 5576b9caaf814564830f95561227e4f04ee87b31da22c1371aab155cbf7ce395 2694800 calamares_0+git20141127.99.tar.xz
+ 2e6e2f232ed7ffe52369928ebdf5436d90feb37840286ffba79e87d57a43a2e9 1698924 calamares_0+git20141127.99_amd64.deb
+ 8dd926080ed7bad2e2439e37e49ce12d5f1357c5041b7da4d860a1041f878a8a 12835902 calamares-dbg_0+git20141127.99_amd64.deb
+Files:
+ 05fd8f3ffe8f362c5ef9bad2f936a56e 1106 devel optional calamares_0+git20141127.99.dsc
+ 097e55c81abd8e5f30bb2eed90c2c1e9 2694800 devel optional calamares_0+git20141127.99.tar.xz
+ 827fb3b12534241e119815d331e8197b 1698924 devel optional calamares_0+git20141127.99_amd64.deb
+ e6f8ce70f564d1f68cb57758b15b13e3 12835902 debug optional calamares-dbg_0+git20141127.99_amd64.deb`

deb/contents.go

@@ -0,0 +1,75 @@
+package deb
+
+import (
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"io"
+	"sort"
+	"strings"
+)
+
+// ContentsIndex calculates mapping from files to packages, with sorting and aggregation
+type ContentsIndex struct {
+	index map[string][]*Package
+}
+
+// NewContentsIndex creates empty ContentsIndex
+func NewContentsIndex() *ContentsIndex {
+	return &ContentsIndex{
+		index: make(map[string][]*Package),
+	}
+}
+
+// Push adds package to contents index, calculating package contents as required
+func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool) {
+	contents := p.Contents(packagePool)
+
+	for _, path := range contents {
+		index.index[path] = append(index.index[path], p)
+	}
+}
+
+// Empty checks whether index contains no packages
+func (index *ContentsIndex) Empty() bool {
+	return len(index.index) == 0
+}
+
+// WriteTo dumps sorted mapping of files to qualified package names
+func (index *ContentsIndex) WriteTo(w io.Writer) (int64, error) {
+	var n int64
+
+	paths := make([]string, len(index.index))
+
+	i := 0
+	for path := range index.index {
+		paths[i] = path
+		i++
+	}
+
+	sort.Strings(paths)
+
+	nn, err := fmt.Fprintf(w, "%s %s\n", "FILE", "LOCATION")
+	n += int64(nn)
+	if err != nil {
+		return n, err
+	}
+
+	for _, path := range paths {
+		packages := index.index[path]
+		parts := make([]string, 0, len(packages))
+		for i := range packages {
+			name := packages[i].QualifiedName()
+			if !utils.StrSliceHasItem(parts, name) {
+				parts = append(parts, name)
+			}
+		}
+		nn, err = fmt.Fprintf(w, "%s %s\n", path, strings.Join(parts, ","))
+		n += int64(nn)
+		if err != nil {
+			return n, err
+		}
+	}
+
+	return n, nil
+}

deb/deb.go

@@ -2,11 +2,13 @@ package deb
 
 import (
 	"archive/tar"
-	"bufio"
+	"compress/bzip2"
 	"compress/gzip"
 	"fmt"
 	"github.com/mkrautz/goar"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/go-xz"
+	"github.com/smira/lzma"
 	"io"
 	"os"
 	"strings"
@@ -24,16 +26,16 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	for {
 		header, err := library.Next()
 		if err == io.EOF {
-			return nil, fmt.Errorf("unable to find control.tar.gz part")
+			return nil, fmt.Errorf("unable to find control.tar.gz part in package %s", packageFile)
 		}
 		if err != nil {
-			return nil, fmt.Errorf("unable to read .deb archive: %s", err)
+			return nil, fmt.Errorf("unable to read .deb archive %s: %s", packageFile, err)
 		}
 
 		if header.Name == "control.tar.gz" {
 			ungzip, err := gzip.NewReader(library)
 			if err != nil {
-				return nil, fmt.Errorf("unable to ungzip: %s", err)
+				return nil, fmt.Errorf("unable to ungzip control file from %s. Error: %s", packageFile, err)
 			}
 			defer ungzip.Close()
 
@@ -41,15 +43,15 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 			for {
 				tarHeader, err := untar.Next()
 				if err == io.EOF {
-					return nil, fmt.Errorf("unable to find control file")
+					return nil, fmt.Errorf("unable to find control file in %s", packageFile)
 				}
 				if err != nil {
-					return nil, fmt.Errorf("unable to read .tar archive: %s", err)
+					return nil, fmt.Errorf("unable to read .tar archive from %s. Error: %s", packageFile, err)
 				}
 
 				if tarHeader.Name == "./control" || tarHeader.Name == "control" {
 					reader := NewControlFileReader(untar)
-					stanza, err := reader.ReadStanza()
+					stanza, err := reader.ReadStanza(false)
 					if err != nil {
 						return nil, err
 					}
@@ -69,16 +71,16 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 	}
 	defer file.Close()
 
-	line, err := bufio.NewReader(file).ReadString('\n')
+	isClearSigned, err := verifier.IsClearSigned(file)
+	file.Seek(0, 0)
+
 	if err != nil {
 		return nil, err
 	}
 
-	file.Seek(0, 0)
-
 	var text *os.File
 
-	if strings.Index(line, "BEGIN PGP SIGN") != -1 {
+	if isClearSigned {
 		text, err = verifier.ExtractClearsigned(file)
 		if err != nil {
 			return nil, err
@@ -89,7 +91,7 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 	}
 
 	reader := NewControlFileReader(text)
-	stanza, err := reader.ReadStanza()
+	stanza, err := reader.ReadStanza(false)
 	if err != nil {
 		return nil, err
 	}
@@ -97,3 +99,75 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 	return stanza, nil
 
 }
+
+// GetContentsFromDeb returns list of files installed by .deb package
+func GetContentsFromDeb(packageFile string) ([]string, error) {
+	file, err := os.Open(packageFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	library := ar.NewReader(file)
+	for {
+		header, err := library.Next()
+		if err == io.EOF {
+			return nil, fmt.Errorf("unable to find data.tar.* part in %s", packageFile)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("unable to read .deb archive from %s: %s", packageFile, err)
+		}
+
+		if strings.HasPrefix(header.Name, "data.tar") {
+			var tarInput io.Reader
+
+			switch header.Name {
+			case "data.tar":
+				tarInput = library
+			case "data.tar.gz":
+				ungzip, err := gzip.NewReader(library)
+				if err != nil {
+					return nil, fmt.Errorf("unable to ungzip data.tar.gz from %s: %s", packageFile, err)
+				}
+				defer ungzip.Close()
+				tarInput = ungzip
+			case "data.tar.bz2":
+				tarInput = bzip2.NewReader(library)
+			case "data.tar.xz":
+				unxz, err := xz.NewReader(library)
+				if err != nil {
+					return nil, fmt.Errorf("unable to unxz data.tar.xz from %s: %s", packageFile, err)
+				}
+				defer unxz.Close()
+				tarInput = unxz
+			case "data.tar.lzma":
+				unlzma := lzma.NewReader(library)
+				defer unlzma.Close()
+				tarInput = unlzma
+			default:
+				return nil, fmt.Errorf("unsupported tar compression in %s: %s", packageFile, header.Name)
+			}
+
+			untar := tar.NewReader(tarInput)
+			var results []string
+			for {
+				tarHeader, err := untar.Next()
+				if err == io.EOF {
+					return results, nil
+				}
+				if err != nil {
+					return nil, fmt.Errorf("unable to read .tar archive from %s: %s", packageFile, err)
+				}
+
+				if tarHeader.Typeflag == tar.TypeDir {
+					continue
+				}
+
+				if strings.HasPrefix(tarHeader.Name, "./") {
+					tarHeader.Name = tarHeader.Name[2:]
+				}
+				results = append(results, tarHeader.Name)
+			}
+		}
+	}
+}

deb/deb_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 type DebSuite struct {
-	debFile, dscFile, dscFileNoSign string
+	debFile, debFile2, dscFile, dscFileNoSign string
 }
 
 var _ = Suite(&DebSuite{})
@@ -17,6 +17,7 @@ var _ = Suite(&DebSuite{})
 func (s *DebSuite) SetUpSuite(c *C) {
 	_, _File, _, _ := runtime.Caller(0)
 	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	s.debFile2 = filepath.Join(filepath.Dir(_File), "../system/changes/hardlink_0.2.1_amd64.deb")
 	s.dscFile = filepath.Join(filepath.Dir(_File), "../system/files/pyspi_0.6.1-1.3.dsc")
 	s.dscFileNoSign = filepath.Join(filepath.Dir(_File), "../system/files/pyspi-0.6.1-1.3.stripped.dsc")
 }
@@ -27,7 +28,7 @@ func (s *DebSuite) TestGetControlFileFromDeb(c *C) {
 
 	_, _File, _, _ := runtime.Caller(0)
 	_, err = GetControlFileFromDeb(_File)
-	c.Check(err, ErrorMatches, "unable to read .deb archive: ar: missing global header")
+	c.Check(err, ErrorMatches, "^.+ar: missing global header")
 
 	st, err := GetControlFileFromDeb(s.debFile)
 	c.Check(err, IsNil)
@@ -55,3 +56,15 @@ func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
 	c.Check(st["Version"], Equals, "0.6.1-1.4")
 	c.Check(st["Source"], Equals, "pyspi")
 }
+
+func (s *DebSuite) TestGetContentsFromDeb(c *C) {
+	contents, err := GetContentsFromDeb(s.debFile)
+	c.Check(err, IsNil)
+	c.Check(contents, DeepEquals, []string{"usr/share/doc/libboost-program-options-dev/changelog.gz",
+		"usr/share/doc/libboost-program-options-dev/copyright"})
+
+	contents, err = GetContentsFromDeb(s.debFile2)
+	c.Check(err, IsNil)
+	c.Check(contents, DeepEquals, []string{"usr/bin/hardlink", "usr/share/man/man1/hardlink.1.gz",
+		"usr/share/doc/hardlink/changelog.gz", "usr/share/doc/hardlink/copyright", "usr/share/doc/hardlink/NEWS.Debian.gz"})
+}

deb/format.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"io"
 	"strings"
+	"unicode"
 )
 
 // Stanza or paragraph of Debian control file
@@ -29,6 +30,7 @@ var (
 		"MD5Sum",
 		"SHA1",
 		"SHA256",
+		"SHA512",
 	}
 
 	canonicalOrderBinary = []string{
@@ -58,6 +60,7 @@ var (
 		"MD5sum",
 		"SHA1",
 		"SHA256",
+		"SHA512",
 		"Description",
 	}
 
@@ -91,16 +94,46 @@ func (s Stanza) Copy() (result Stanza) {
 	return
 }
 
-// Write single field from Stanza to writer
-func writeField(w *bufio.Writer, field, value string) (err error) {
-	_, multiline := multilineFields[field]
+func isMultilineField(field string, isRelease bool) bool {
+	switch field {
+	case "Description":
+		return true
+	case "Files":
+		return true
+	case "Changes":
+		return true
+	case "Checksums-Sha1":
+		return true
+	case "Checksums-Sha256":
+		return true
+	case "Checksums-Sha512":
+		return true
+	case "Package-List":
+		return true
+	case "MD5Sum":
+		return isRelease
+	case "SHA1":
+		return isRelease
+	case "SHA256":
+		return isRelease
+	case "SHA512":
+		return isRelease
+	}
+	return false
+}
 
-	if !multiline {
+// Write single field from Stanza to writer
+func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) {
+	if !isMultilineField(field, isRelease) {
 		_, err = w.WriteString(field + ": " + value + "\n")
 	} else {
 		if !strings.HasSuffix(value, "\n") {
 			value = value + "\n"
 		}
+
+		if field != "Description" {
+			value = "\n" + value
+		}
 		_, err = w.WriteString(field + ":" + value)
 	}
 
@@ -121,7 +154,7 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 		value, ok := s[field]
 		if ok {
 			delete(s, field)
-			err := writeField(w, field, value)
+			err := writeField(w, field, value, isRelease)
 			if err != nil {
 				return err
 			}
@@ -129,7 +162,7 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 	}
 
 	for field, value := range s {
-		err := writeField(w, field, value)
+		err := writeField(w, field, value, isRelease)
 		if err != nil {
 			return err
 		}
@@ -143,18 +176,33 @@ var (
 	ErrMalformedStanza = errors.New("malformed stanza syntax")
 )
 
-var multilineFields = make(map[string]bool)
+func canonicalCase(field string) string {
+	upper := strings.ToUpper(field)
+	switch upper {
+	case "SHA1", "SHA256", "SHA512":
+		return upper
+	case "MD5SUM":
+		return "MD5Sum"
+	case "NOTAUTOMATIC":
+		return "NotAutomatic"
+	case "BUTAUTOMATICUPGRADES":
+		return "ButAutomaticUpgrades"
+	}
 
-func init() {
-	multilineFields["Description"] = true
-	multilineFields["Files"] = true
-	multilineFields["Changes"] = true
-	multilineFields["Checksums-Sha1"] = true
-	multilineFields["Checksums-Sha256"] = true
-	multilineFields["Package-List"] = true
-	multilineFields["SHA256"] = true
-	multilineFields["SHA1"] = true
-	multilineFields["MD5Sum"] = true
+	startOfWord := true
+
+	return strings.Map(func(r rune) rune {
+		if startOfWord {
+			startOfWord = false
+			return unicode.ToUpper(r)
+		}
+
+		if r == '-' {
+			startOfWord = true
+		}
+
+		return unicode.ToLower(r)
+	}, field)
 }
 
 // ControlFileReader implements reading of control files stanza by stanza
@@ -168,7 +216,7 @@ func NewControlFileReader(r io.Reader) *ControlFileReader {
 }
 
 // ReadStanza reeads one stanza from control file
-func (c *ControlFileReader) ReadStanza() (Stanza, error) {
+func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
 	stanza := make(Stanza, 32)
 	lastField := ""
 	lastFieldMultiline := false
@@ -188,15 +236,15 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 			if lastFieldMultiline {
 				stanza[lastField] += line + "\n"
 			} else {
-				stanza[lastField] += strings.TrimSpace(line)
+				stanza[lastField] += " " + strings.TrimSpace(line)
 			}
 		} else {
 			parts := strings.SplitN(line, ":", 2)
 			if len(parts) != 2 {
 				return nil, ErrMalformedStanza
 			}
-			lastField = parts[0]
-			_, lastFieldMultiline = multilineFields[lastField]
+			lastField = canonicalCase(parts[0])
+			lastFieldMultiline = isMultilineField(lastField, isRelease)
 			if lastFieldMultiline {
 				stanza[lastField] = parts[1]
 				if parts[1] != "" {

deb/format_test.go

@@ -84,18 +84,18 @@ func (s *ControlFileSuite) SetUpTest(c *C) {
 func (s *ControlFileSuite) TestReadStanza(c *C) {
 	r := NewControlFileReader(s.reader)
 
-	stanza1, err := r.ReadStanza()
+	stanza1, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
-	stanza2, err := r.ReadStanza()
+	stanza2, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
-	stanza3, err := r.ReadStanza()
+	stanza3, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 	c.Assert(stanza3, IsNil)
 
 	c.Check(stanza1["Format"], Equals, "3.0 (quilt)")
-	c.Check(stanza1["Build-Depends"], Equals, "debhelper (>= 8),bash-completion (>= 1:1.1-3),libcurl4-nss-dev, libreadline-dev, libxml2-dev, libpcre3-dev, liboauth-dev, xsltproc, docbook-xsl, docbook-xml, dh-autoreconf")
+	c.Check(stanza1["Build-Depends"], Equals, "debhelper (>= 8), bash-completion (>= 1:1.1-3), libcurl4-nss-dev, libreadline-dev, libxml2-dev, libpcre3-dev, liboauth-dev, xsltproc, docbook-xsl, docbook-xml, dh-autoreconf")
 	c.Check(stanza1["Files"], Equals, " 3d5f65778bf3f89be03c313b0024b62c 1980 bti_032-1.dsc\n"+
 		" 1e0d0b693fdeebec268004ba41701baf 59773 bti_032.orig.tar.gz\n"+" ac1229a6d685023aeb8fcb0806324aa8 5065 bti_032-1.debian.tar.gz\n")
 	c.Check(len(stanza2), Equals, 20)
@@ -103,12 +103,12 @@ func (s *ControlFileSuite) TestReadStanza(c *C) {
 
 func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 	r := NewControlFileReader(s.reader)
-	stanza, err := r.ReadStanza()
+	stanza, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
 	buf := &bytes.Buffer{}
 	w := bufio.NewWriter(buf)
-	err = stanza.Copy().WriteTo(w, false, false)
+	err = stanza.Copy().WriteTo(w, true, false)
 	c.Assert(err, IsNil)
 	err = w.Flush()
 	c.Assert(err, IsNil)
@@ -116,19 +116,31 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 	str := buf.String()
 
 	r = NewControlFileReader(buf)
-	stanza2, err := r.ReadStanza()
+	stanza2, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
 	c.Assert(stanza2, DeepEquals, stanza)
 	c.Assert(strings.HasPrefix(str, "Package: "), Equals, true)
 }
 
+func (s *ControlFileSuite) TestCanonicalCase(c *C) {
+	c.Check(canonicalCase("Package"), Equals, "Package")
+	c.Check(canonicalCase("package"), Equals, "Package")
+	c.Check(canonicalCase("pAckaGe"), Equals, "Package")
+	c.Check(canonicalCase("MD5Sum"), Equals, "MD5Sum")
+	c.Check(canonicalCase("SHA1"), Equals, "SHA1")
+	c.Check(canonicalCase("SHA256"), Equals, "SHA256")
+	c.Check(canonicalCase("Package-List"), Equals, "Package-List")
+	c.Check(canonicalCase("package-list"), Equals, "Package-List")
+	c.Check(canonicalCase("packaGe-lIst"), Equals, "Package-List")
+}
+
 func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
 	for i := 0; i < c.N; i++ {
 		reader := bytes.NewBufferString(controlFile)
 		r := NewControlFileReader(reader)
 		for {
-			s, e := r.ReadStanza()
+			s, e := r.ReadStanza(false)
 			if s == nil && e == nil {
 				break
 			}

deb/graph.go

@@ -1,8 +1,8 @@
 package deb
 
 import (
-	"code.google.com/p/gographviz"
 	"fmt"
+	"github.com/awalterschulze/gographviz"
 	"strings"
 )
 

deb/import.go

@@ -10,7 +10,7 @@ import (
 )
 
 // CollectPackageFiles walks filesystem collecting all candidates for package files
-func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, failedFiles []string, err error) {
+func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, failedFiles []string) {
 	for _, location := range locations {
 		info, err2 := os.Stat(location)
 		if err2 != nil {
@@ -28,15 +28,21 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 				}
 
 				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
-					strings.HasSuffix(info.Name(), ".dsc") {
+					strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
 					packageFiles = append(packageFiles, path)
 				}
 
 				return nil
 			})
+
+			if err2 != nil {
+				reporter.Warning("Unable to process %s: %s", location, err2)
+				failedFiles = append(failedFiles, location)
+				continue
+			}
 		} else {
 			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
-				strings.HasSuffix(info.Name(), ".dsc") {
+				strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
 				packageFiles = append(packageFiles, location)
 			} else {
 				reporter.Warning("Unknown file extension: %s", location)
@@ -53,7 +59,7 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 
 // ImportPackageFiles imports files into local repository
 func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier utils.Verifier,
-	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter) (processedFiles []string, failedFiles []string, err error) {
+	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter, restriction PackageQuery) (processedFiles []string, failedFiles []string, err error) {
 	if forceReplace {
 		list.PrepareIndex()
 	}
@@ -91,6 +97,24 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
+		if p.Name == "" {
+			reporter.Warning("Empty package name on %s", file)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		if p.Version == "" {
+			reporter.Warning("Empty version on %s", file)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		if p.Architecture == "" {
+			reporter.Warning("Empty architecture on %s", file)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
 		var checksums utils.ChecksumInfo
 		checksums, err = utils.ChecksumsForFile(file)
 		if err != nil {
@@ -132,6 +156,12 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
+		if restriction != nil && !restriction.Matches(p) {
+			reporter.Warning("%s has been ignored as it doesn't match restriction", p)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
 		err = collection.Update(p)
 		if err != nil {
 			reporter.Warning("Unable to save package %s: %s", p, err)

deb/index_files.go

@@ -24,6 +24,7 @@ type indexFile struct {
 	parent       *indexFiles
 	discardable  bool
 	compressable bool
+	onlyGzip     bool
 	signable     bool
 	relativePath string
 	tempFilename string
@@ -73,6 +74,9 @@ func (file *indexFile) Finalize(signer utils.Signer) error {
 	exts := []string{""}
 	if file.compressable {
 		exts = append(exts, ".gz", ".bz2")
+		if file.onlyGzip {
+			exts = []string{".gz"}
+		}
 	}
 
 	for _, ext := range exts {
@@ -215,6 +219,36 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 	return file
 }
 
+func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *indexFile {
+	if arch == "source" {
+		udeb = false
+	}
+	key := fmt.Sprintf("ci-%s-%s-%v", component, arch, udeb)
+	file, ok := files.indexes[key]
+	if !ok {
+		var relativePath string
+
+		if udeb {
+			relativePath = filepath.Join(component, fmt.Sprintf("Contents-udeb-%s", arch))
+		} else {
+			relativePath = filepath.Join(component, fmt.Sprintf("Contents-%s", arch))
+		}
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  true,
+			compressable: true,
+			onlyGzip:     true,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
 func (files *indexFiles) ReleaseFile() *indexFile {
 	return &indexFile{
 		parent:       files,

deb/list.go

@@ -36,6 +36,10 @@ type PackageList struct {
 	packagesIndex []*Package
 	// Map of packages for each virtual package (provides)
 	providesIndex map[string][]*Package
+	// Package key generation function
+	keyFunc func(p *Package) string
+	// Allow duplicates?
+	duplicatesAllowed bool
 }
 
 // PackageConflictError means that package can't be added to the list due to error
@@ -49,9 +53,35 @@ var (
 	_ PackageCatalog = &PackageList{}
 )
 
-// NewPackageList creates empty package list
+func packageShortKey(p *Package) string {
+	return string(p.ShortKey(""))
+}
+
+func packageFullKey(p *Package) string {
+	return string(p.Key(""))
+}
+
+// NewPackageList creates empty package list without duplicate package
 func NewPackageList() *PackageList {
-	return &PackageList{packages: make(map[string]*Package, 1000)}
+	return NewPackageListWithDuplicates(false, 1000)
+}
+
+func NewPackageListWithDuplicates(duplicates bool, capacity int) *PackageList {
+	if capacity == 0 {
+		capacity = 1000
+	}
+
+	result := &PackageList{
+		packages:          make(map[string]*Package, capacity),
+		duplicatesAllowed: duplicates,
+		keyFunc:           packageShortKey,
+	}
+
+	if duplicates {
+		result.keyFunc = packageFullKey
+	}
+
+	return result
 }
 
 // NewPackageListFromRefList loads packages list from PackageRefList
@@ -61,7 +91,7 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 		return NewPackageList(), nil
 	}
 
-	result := &PackageList{packages: make(map[string]*Package, reflist.Len())}
+	result := NewPackageListWithDuplicates(false, reflist.Len())
 
 	if progress != nil {
 		progress.InitBar(int64(reflist.Len()), false)
@@ -91,7 +121,7 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 
 // Add appends package to package list, additionally checking for uniqueness
 func (l *PackageList) Add(p *Package) error {
-	key := string(p.ShortKey(""))
+	key := l.keyFunc(p)
 	existing, ok := l.packages[key]
 	if ok {
 		if !existing.Equals(p) {
@@ -170,7 +200,7 @@ func (l *PackageList) Append(pl *PackageList) error {
 
 // Remove removes package from the list, and updates index when required
 func (l *PackageList) Remove(p *Package) {
-	delete(l.packages, string(p.ShortKey("")))
+	delete(l.packages, l.keyFunc(p))
 	if l.indexed {
 		for _, provides := range p.Provides {
 			for i, pkg := range l.providesIndex[provides] {
@@ -252,7 +282,7 @@ func depSliceDeduplicate(s []Dependency) []Dependency {
 
 // VerifyDependencies looks for missing dependencies in package list.
 //
-// Analysis would be peformed for each architecture, in specified sources
+// Analysis would be performed for each architecture, in specified sources
 func (l *PackageList) VerifyDependencies(options int, architectures []string, sources *PackageList, progress aptly.Progress) ([]Dependency, error) {
 	l.PrepareIndex()
 	missing := make([]Dependency, 0, 128)
@@ -365,7 +395,7 @@ func (l *PackageList) PrepareIndex() {
 
 // Scan searches package index using full scan
 func (l *PackageList) Scan(q PackageQuery) (result *PackageList) {
-	result = NewPackageList()
+	result = NewPackageListWithDuplicates(l.duplicatesAllowed, 0)
 	for _, pkg := range l.packages {
 		if q.Matches(pkg) {
 			result.Add(pkg)
@@ -382,7 +412,7 @@ func (l *PackageList) SearchSupported() bool {
 
 // SearchByKey looks up package by exact key reference
 func (l *PackageList) SearchByKey(arch, name, version string) (result *PackageList) {
-	result = NewPackageList()
+	result = NewPackageListWithDuplicates(l.duplicatesAllowed, 0)
 
 	pkg := l.packages["P"+arch+" "+name+" "+version]
 	if pkg != nil {

deb/list_test.go

@@ -379,6 +379,20 @@ func (s *PackageListSuite) TestFilter(c *C) {
 		&FieldQuery{Field: "$Architecture", Relation: VersionRegexp, Value: "i.*6", Regexp: regexp.MustCompile("i.*6")}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")},
+		&NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}},
+	}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}},
+		&FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")},
+	}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386")
 }
 
 func (s *PackageListSuite) TestVerifyDependencies(c *C) {

deb/local.go

@@ -2,9 +2,9 @@ package deb
 
 import (
 	"bytes"
-	"code.google.com/p/go-uuid/uuid"
 	"fmt"
 	"github.com/smira/aptly/database"
+	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
 	"log"
 	"sync"
@@ -22,6 +22,8 @@ type LocalRepo struct {
 	DefaultDistribution string `codec:",omitempty"`
 	// DefaultComponent
 	DefaultComponent string `codec:",omitempty"`
+	// Uploaders configuration
+	Uploaders *Uploaders `code:",omitempty" json:"-"`
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 }

deb/package.go

@@ -32,9 +32,10 @@ type Package struct {
 	// Is this >= 0.6 package?
 	V06Plus bool
 	// Offload fields
-	deps  *PackageDependencies
-	extra *Stanza
-	files *PackageFiles
+	deps     *PackageDependencies
+	extra    *Stanza
+	files    *PackageFiles
+	contents []string
 	// Mother collection
 	collection *PackageCollection
 }
@@ -75,6 +76,7 @@ func NewPackageFromControlFile(input Stanza) *Package {
 			MD5:    strings.TrimSpace(md5),
 			SHA1:   strings.TrimSpace(input["SHA1"]),
 			SHA256: strings.TrimSpace(input["SHA256"]),
+			SHA512: strings.TrimSpace(input["SHA512"]),
 		},
 	}})
 
@@ -83,6 +85,7 @@ func NewPackageFromControlFile(input Stanza) *Package {
 	delete(input, "MD5Sum")
 	delete(input, "SHA1")
 	delete(input, "SHA256")
+	delete(input, "SHA512")
 	delete(input, "Size")
 
 	depends := &PackageDependencies{}
@@ -114,62 +117,20 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 	delete(input, "Version")
 	delete(input, "Architecture")
 
+	var err error
+
 	files := make(PackageFiles, 0, 3)
-
-	parseSums := func(field string, setter func(sum *utils.ChecksumInfo, data string)) error {
-		for _, line := range strings.Split(input[field], "\n") {
-			line = strings.TrimSpace(line)
-			if line == "" {
-				continue
-			}
-			parts := strings.Fields(line)
-
-			if len(parts) != 3 {
-				return fmt.Errorf("unparseable hash sum line: %#v", line)
-			}
-
-			size, err := strconv.ParseInt(parts[1], 10, 64)
-			if err != nil {
-				return fmt.Errorf("unable to parse size: %s", err)
-			}
-
-			filename := filepath.Base(parts[2])
-
-			found := false
-			pos := 0
-			for i, file := range files {
-				if file.Filename == filename {
-					found = true
-					pos = i
-					break
-				}
-			}
-
-			if !found {
-				files = append(files, PackageFile{Filename: filename, downloadPath: input["Directory"]})
-				pos = len(files) - 1
-			}
-
-			files[pos].Checksums.Size = size
-			setter(&files[pos].Checksums, parts[0])
-		}
-
-		delete(input, field)
-
-		return nil
-	}
-
-	err := parseSums("Files", func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
+	files, err = files.ParseSumFields(input)
 	if err != nil {
 		return nil, err
 	}
-	err = parseSums("Checksums-Sha1", func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
-	if err != nil {
-		return nil, err
-	}
-	err = parseSums("Checksums-Sha256", func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
-	if err != nil {
-		return nil, err
+
+	delete(input, "Files")
+	delete(input, "Checksums-Sha1")
+	delete(input, "Checksums-Sha256")
+
+	for i := range files {
+		files[i].downloadPath = input["Directory"]
 	}
 
 	result.UpdateFiles(files)
@@ -211,14 +172,19 @@ func (p *Package) String() string {
 	return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.Architecture)
 }
 
-// MarshalJSON implements json.Marshaller interface
-func (p *Package) MarshalJSON() ([]byte, error) {
+// ExtendedStanza returns package stanza enhanced with aptly-specific fields
+func (p *Package) ExtendedStanza() Stanza {
 	stanza := p.Stanza()
 	stanza["FilesHash"] = fmt.Sprintf("%08x", p.FilesHash)
 	stanza["Key"] = string(p.Key(""))
 	stanza["ShortKey"] = string(p.ShortKey(""))
 
-	return json.Marshal(stanza)
+	return stanza
+}
+
+// MarshalJSON implements json.Marshaller interface
+func (p *Package) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.ExtendedStanza())
 }
 
 // GetField returns fields from package
@@ -336,6 +302,21 @@ func (p *Package) MatchesDependency(dep Dependency) bool {
 	panic("unknown relation")
 }
 
+// GetName returns package name
+func (p *Package) GetName() string {
+	return p.Name
+}
+
+// GetVersion returns package version
+func (p *Package) GetVersion() string {
+	return p.Version
+}
+
+// GetArchitecture returns package arch
+func (p *Package) GetArchitecture() string {
+	return p.Architecture
+}
+
 // GetDependencies compiles list of dependenices by flags from options
 func (p *Package) GetDependencies(options int) (dependencies []string) {
 	deps := p.Deps()
@@ -372,6 +353,16 @@ func (p *Package) GetDependencies(options int) (dependencies []string) {
 	return
 }
 
+// QualifiedName returns [$SECTION/]$NAME
+func (p *Package) QualifiedName() string {
+	section := p.Extra()["Section"]
+	if section != "" {
+		return section + "/" + p.Name
+	}
+
+	return p.Name
+}
+
 // Extra returns Stanza of extra fields (it may load it from collection)
 func (p *Package) Extra() Stanza {
 	if p.extra == nil {
@@ -410,6 +401,35 @@ func (p *Package) Files() PackageFiles {
 	return *p.files
 }
 
+// Contents returns cached package contents
+func (p *Package) Contents(packagePool aptly.PackagePool) []string {
+	if p.IsSource {
+		return nil
+	}
+
+	return p.collection.loadContents(p, packagePool)
+}
+
+// CalculateContents looks up contents in package file
+func (p *Package) CalculateContents(packagePool aptly.PackagePool) []string {
+	if p.IsSource {
+		return nil
+	}
+
+	file := p.Files()[0]
+	path, err := packagePool.Path(file.Filename, file.Checksums.MD5)
+	if err != nil {
+		panic(err)
+	}
+
+	contents, err := GetContentsFromDeb(path)
+	if err != nil {
+		panic(err)
+	}
+
+	return contents
+}
+
 // UpdateFiles saves new state of files
 func (p *Package) UpdateFiles(files PackageFiles) {
 	p.files = &files
@@ -432,7 +452,7 @@ func (p *Package) Stanza() (result Stanza) {
 	}
 
 	if p.IsSource {
-		md5, sha1, sha256 := make([]string, 0), make([]string, 0), make([]string, 0)
+		md5, sha1, sha256, sha512 := []string{}, []string{}, []string{}, []string{}
 
 		for _, f := range p.Files() {
 			if f.Checksums.MD5 != "" {
@@ -444,11 +464,21 @@ func (p *Package) Stanza() (result Stanza) {
 			if f.Checksums.SHA256 != "" {
 				sha256 = append(sha256, fmt.Sprintf(" %s %d %s\n", f.Checksums.SHA256, f.Checksums.Size, f.Filename))
 			}
+			if f.Checksums.SHA512 != "" {
+				sha512 = append(sha512, fmt.Sprintf(" %s %d %s\n", f.Checksums.SHA512, f.Checksums.Size, f.Filename))
+			}
 		}
 
 		result["Files"] = strings.Join(md5, "")
-		result["Checksums-Sha1"] = strings.Join(sha1, "")
-		result["Checksums-Sha256"] = strings.Join(sha256, "")
+		if len(sha1) > 0 {
+			result["Checksums-Sha1"] = strings.Join(sha1, "")
+		}
+		if len(sha256) > 0 {
+			result["Checksums-Sha256"] = strings.Join(sha256, "")
+		}
+		if len(sha512) > 0 {
+			result["Checksums-Sha512"] = strings.Join(sha512, "")
+		}
 	} else {
 		f := p.Files()[0]
 		result["Filename"] = f.DownloadURL()
@@ -456,10 +486,13 @@ func (p *Package) Stanza() (result Stanza) {
 			result["MD5sum"] = f.Checksums.MD5
 		}
 		if f.Checksums.SHA1 != "" {
-			result["SHA1"] = " " + f.Checksums.SHA1
+			result["SHA1"] = f.Checksums.SHA1
 		}
 		if f.Checksums.SHA256 != "" {
-			result["SHA256"] = " " + f.Checksums.SHA256
+			result["SHA256"] = f.Checksums.SHA256
+		}
+		if f.Checksums.SHA512 != "" {
+			result["SHA512"] = f.Checksums.SHA512
 		}
 		result["Size"] = fmt.Sprintf("%d", f.Checksums.Size)
 	}

deb/package_collection.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"bytes"
 	"fmt"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/ugorji/go/codec"
 	"path/filepath"
@@ -10,9 +11,8 @@ import (
 
 // PackageCollection does management of packages in DB
 type PackageCollection struct {
-	db           database.Storage
-	encodeBuffer bytes.Buffer
-	codecHandle  *codec.MsgpackHandle
+	db          database.Storage
+	codecHandle *codec.MsgpackHandle
 }
 
 // Verify interface
@@ -161,45 +161,82 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 	return files
 }
 
+// loadContents loads or calculates and saves package contents
+func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.PackagePool) []string {
+	encoded, err := collection.db.Get(p.Key("xC"))
+	if err == nil {
+		contents := []string{}
+
+		decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
+		err = decoder.Decode(&contents)
+		if err != nil {
+			panic("unable to decode contents")
+		}
+
+		return contents
+	}
+
+	if err != database.ErrNotFound {
+		panic("unable to load contents")
+	}
+
+	contents := p.CalculateContents(packagePool)
+
+	var buf bytes.Buffer
+	err = codec.NewEncoder(&buf, collection.codecHandle).Encode(contents)
+	if err != nil {
+		panic("unable to encode contents")
+	}
+
+	err = collection.db.Put(p.Key("xC"), buf.Bytes())
+	if err != nil {
+		panic("unable to save contents")
+	}
+
+	return contents
+}
+
 // Update adds or updates information about package in DB checking for conficts first
 func (collection *PackageCollection) Update(p *Package) error {
-	encoder := codec.NewEncoder(&collection.encodeBuffer, collection.codecHandle)
+	var encodeBuffer bytes.Buffer
 
-	collection.encodeBuffer.Reset()
-	collection.encodeBuffer.WriteByte(0xc1)
-	collection.encodeBuffer.WriteByte(0x1)
+	encoder := codec.NewEncoder(&encodeBuffer, collection.codecHandle)
+
+	encodeBuffer.Reset()
+	encodeBuffer.WriteByte(0xc1)
+	encodeBuffer.WriteByte(0x1)
 	err := encoder.Encode(p)
 	if err != nil {
 		return err
 	}
 
-	err = collection.db.Put(p.Key(""), collection.encodeBuffer.Bytes())
+	err = collection.db.Put(p.Key(""), encodeBuffer.Bytes())
 	if err != nil {
 		return err
 	}
 
 	// Encode offloaded fields one by one
 	if p.files != nil {
-		collection.encodeBuffer.Reset()
+		encodeBuffer.Reset()
 		err = encoder.Encode(*p.files)
 		if err != nil {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xF"), collection.encodeBuffer.Bytes())
+		err = collection.db.Put(p.Key("xF"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
 	}
 
 	if p.deps != nil {
-		collection.encodeBuffer.Reset()
+		encodeBuffer.Reset()
 		err = encoder.Encode(*p.deps)
 		if err != nil {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xD"), collection.encodeBuffer.Bytes())
+		err = collection.db.Put(p.Key("xD"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
@@ -208,13 +245,13 @@ func (collection *PackageCollection) Update(p *Package) error {
 	}
 
 	if p.extra != nil {
-		collection.encodeBuffer.Reset()
+		encodeBuffer.Reset()
 		err = encoder.Encode(*p.extra)
 		if err != nil {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xE"), collection.encodeBuffer.Bytes())
+		err = collection.db.Put(p.Key("xE"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
@@ -245,7 +282,7 @@ func (collection *PackageCollection) DeleteByKey(key []byte) error {
 
 // Scan does full scan on all the packages
 func (collection *PackageCollection) Scan(q PackageQuery) (result *PackageList) {
-	result = NewPackageList()
+	result = NewPackageListWithDuplicates(true, 0)
 
 	for _, key := range collection.db.KeysByPrefix([]byte("P")) {
 		pkg, err := collection.ByKey(key)

deb/package_deps.go

@@ -22,6 +22,12 @@ func parseDependencies(input Stanza, key string) []string {
 
 	delete(input, key)
 
+	value = strings.TrimSpace(value)
+	if value == "" {
+		// empty line is no depdencies
+		return nil
+	}
+
 	result := strings.Split(value, ",")
 	for i := range result {
 		result[i] = strings.TrimSpace(result[i])

deb/package_files.go

@@ -2,12 +2,15 @@ package deb
 
 import (
 	"encoding/binary"
+	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
 	"hash/fnv"
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
+	"strings"
 )
 
 // PackageFile is a single file entry in package
@@ -76,3 +79,71 @@ func (files PackageFiles) Swap(i, j int) {
 func (files PackageFiles) Less(i, j int) bool {
 	return files[i].Filename < files[j].Filename
 }
+
+func (files PackageFiles) parseSumField(input string, setter func(sum *utils.ChecksumInfo, data string)) (PackageFiles, error) {
+	for _, line := range strings.Split(input, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		parts := strings.Fields(line)
+
+		if len(parts) < 3 {
+			return nil, fmt.Errorf("unparseable hash sum line: %#v", line)
+		}
+
+		size, err := strconv.ParseInt(parts[1], 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("unable to parse size: %s", err)
+		}
+
+		filename := filepath.Base(parts[len(parts)-1])
+
+		found := false
+		pos := 0
+		for i, file := range files {
+			if file.Filename == filename {
+				found = true
+				pos = i
+				break
+			}
+		}
+
+		if !found {
+			files = append(files, PackageFile{Filename: filename})
+			pos = len(files) - 1
+		}
+
+		files[pos].Checksums.Size = size
+		setter(&files[pos].Checksums, parts[0])
+	}
+
+	return files, nil
+}
+
+// ParseSumFields populates PackageFiles by parsing stanza checksums fields
+func (files PackageFiles) ParseSumFields(stanza Stanza) (PackageFiles, error) {
+	var err error
+
+	files, err = files.parseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	files, err = files.parseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	files, err = files.parseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	files, err = files.parseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	return files, nil
+}

deb/package_test.go

@@ -22,7 +22,7 @@ func (s *PackageSuite) SetUpTest(c *C) {
 	s.stanza = packageStanza.Copy()
 
 	buf := bytes.NewBufferString(sourcePackageMeta)
-	s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza()
+	s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza(false)
 }
 
 func (s *PackageSuite) TestNewFromPara(c *C) {
@@ -43,7 +43,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 }
 
 func (s *PackageSuite) TestNewUdebFromPara(c *C) {
-	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
 	p := NewUdebPackageFromControlFile(stanza)
 
 	c.Check(p.IsSource, Equals, false)
@@ -125,6 +125,10 @@ func (s *PackageSuite) TestStanza(c *C) {
 	p := NewPackageFromControlFile(s.stanza.Copy())
 	stanza := p.Stanza()
 
+	for k := range s.stanza {
+		c.Check(stanza[k], Equals, s.stanza[k])
+	}
+
 	c.Assert(stanza, DeepEquals, s.stanza)
 
 	p, _ = NewSourcePackageFromControlFile(s.sourceStanza.Copy())
@@ -152,7 +156,7 @@ func (s *PackageSuite) TestGetField(c *C) {
 
 	p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
 
-	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
 	p5 := NewUdebPackageFromControlFile(stanza5)
 
 	c.Check(p.GetField("$Source"), Equals, "alien-arena")
@@ -445,7 +449,7 @@ func (s *PackageSuite) TestVerifyFiles(c *C) {
 	c.Check(result, Equals, true)
 }
 
-var packageStanza = Stanza{"Source": "alien-arena", "Pre-Depends": "dpkg (>= 1.6)", "Suggests": "alien-arena-mars", "Recommends": "aliean-arena-luna", "Depends": "libc6 (>= 2.7), alien-arena-data (>= 7.40)", "Filename": "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", "SHA1": " 46955e48cad27410a83740a21d766ce362364024", "SHA256": " eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", "Priority": "extra", "Maintainer": "Debian Games Team <pkg-games-devel@lists.alioth.debian.org>", "Description": "Common files for Alien Arena client and server ALIEN ARENA is a standalone 3D first person online deathmatch shooter\n crafted from the original source code of Quake II and Quake III, released\n by id Software under the GPL license. With features including 32 bit\n graphics, new particle engine and effects, light blooms, reflective water,\n hi resolution textures and skins, hi poly models, stain maps, ALIEN ARENA\n pushes the envelope of graphical beauty rivaling today's top games.\n .\n This package installs the common files for Alien Arena.\n", "Homepage": "http://red.planetarena.org", "Tag": "role::app-data, role::shared-lib, special::auto-inst-parts", "Installed-Size": "456", "Version": "7.40-2", "Replaces": "alien-arena (<< 7.33-1)", "Size": "187518", "MD5sum": "1e8cba92c41420aa7baa8a5718d67122", "Package": "alien-arena-common", "Section": "contrib/games", "Architecture": "i386"}
+var packageStanza = Stanza{"Source": "alien-arena", "Pre-Depends": "dpkg (>= 1.6)", "Suggests": "alien-arena-mars", "Recommends": "aliean-arena-luna", "Depends": "libc6 (>= 2.7), alien-arena-data (>= 7.40)", "Filename": "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", "SHA1": "46955e48cad27410a83740a21d766ce362364024", "SHA256": "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", "Priority": "extra", "Maintainer": "Debian Games Team <pkg-games-devel@lists.alioth.debian.org>", "Description": "Common files for Alien Arena client and server ALIEN ARENA is a standalone 3D first person online deathmatch shooter\n crafted from the original source code of Quake II and Quake III, released\n by id Software under the GPL license. With features including 32 bit\n graphics, new particle engine and effects, light blooms, reflective water,\n hi resolution textures and skins, hi poly models, stain maps, ALIEN ARENA\n pushes the envelope of graphical beauty rivaling today's top games.\n .\n This package installs the common files for Alien Arena.\n", "Homepage": "http://red.planetarena.org", "Tag": "role::app-data, role::shared-lib, special::auto-inst-parts", "Installed-Size": "456", "Version": "7.40-2", "Replaces": "alien-arena (<< 7.33-1)", "Size": "187518", "MD5sum": "1e8cba92c41420aa7baa8a5718d67122", "Package": "alien-arena-common", "Section": "contrib/games", "Architecture": "i386"}
 
 const sourcePackageMeta = `Package: access-modifier-checker
 Binary: libaccess-modifier-checker-java, libaccess-modifier-checker-java-doc

deb/publish.go

@@ -3,12 +3,12 @@ package deb
 import (
 	"bufio"
 	"bytes"
-	"code.google.com/p/go-uuid/uuid"
 	"encoding/json"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
 	"io/ioutil"
 	"log"
@@ -43,6 +43,8 @@ type PublishedRepo struct {
 	Architectures []string
 	// SourceKind is "local"/"repo"
 	SourceKind string
+	// Skip contents generation
+	SkipContents bool
 
 	// Map of sources by each component: component name -> source UUID
 	Sources map[string]string
@@ -288,10 +290,11 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
 		"SourceKind":    p.SourceKind,
 		"Sources":       sources,
 		"Storage":       p.Storage,
+		"SkipContents":  p.SkipContents,
 	})
 }
 
-// String returns human-readable represenation of PublishedRepo
+// String returns human-readable representation of PublishedRepo
 func (p *PublishedRepo) String() string {
 	var sources = []string{}
 
@@ -525,6 +528,8 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 		list.PrepareIndex()
 
+		contentIndexes := map[string]*ContentsIndex{}
+
 		err = list.ForEachIndexed(func(pkg *Package) error {
 			if progress != nil {
 				progress.AddBar(1)
@@ -550,6 +555,19 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				if pkg.MatchesArchitecture(arch) {
 					var bufWriter *bufio.Writer
 
+					if !p.SkipContents {
+						key := fmt.Sprintf("%s-%v", arch, pkg.IsUdeb)
+
+						contentIndex := contentIndexes[key]
+
+						if contentIndex == nil {
+							contentIndex = NewContentsIndex()
+							contentIndexes[key] = contentIndex
+						}
+
+						contentIndex.Push(pkg, packagePool)
+					}
+
 					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
 					if err != nil {
 						return err
@@ -569,6 +587,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			pkg.files = nil
 			pkg.deps = nil
 			pkg.extra = nil
+			pkg.contents = nil
 
 			return nil
 		})
@@ -577,6 +596,25 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			return fmt.Errorf("unable to process packages: %s", err)
 		}
 
+		for _, arch := range p.Architectures {
+			for _, udeb := range []bool{true, false} {
+				index := contentIndexes[fmt.Sprintf("%s-%v", arch, udeb)]
+				if index == nil || index.Empty() {
+					continue
+				}
+
+				bufWriter, err := indexes.ContentsIndex(component, arch, udeb).BufWriter()
+				if err != nil {
+					return fmt.Errorf("unable to generate contents index: %v", err)
+				}
+
+				_, err = index.WriteTo(bufWriter)
+				if err != nil {
+					return fmt.Errorf("unable to generate contents index: %v", err)
+				}
+			}
+		}
+
 		if progress != nil {
 			progress.ShutdownBar()
 		}
@@ -603,6 +641,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 				var bufWriter *bufio.Writer
 				bufWriter, err = indexes.ReleaseIndex(component, arch, udeb).BufWriter()
+				if err != nil {
+					return fmt.Errorf("unable to get ReleaseIndex writer: %s", err)
+				}
 
 				err = release.WriteTo(bufWriter, false, true)
 				if err != nil {
@@ -629,9 +670,10 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{"source"}), " ")
 	release["Description"] = " Generated by aptly\n"
-	release["MD5Sum"] = "\n"
-	release["SHA1"] = "\n"
-	release["SHA256"] = "\n"
+	release["MD5Sum"] = ""
+	release["SHA1"] = ""
+	release["SHA256"] = ""
+	release["SHA512"] = ""
 
 	release["Components"] = strings.Join(p.Components(), " ")
 
@@ -639,6 +681,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		release["MD5Sum"] += fmt.Sprintf(" %s %8d %s\n", info.MD5, info.Size, path)
 		release["SHA1"] += fmt.Sprintf(" %s %8d %s\n", info.SHA1, info.Size, path)
 		release["SHA256"] += fmt.Sprintf(" %s %8d %s\n", info.SHA256, info.Size, path)
+		release["SHA512"] += fmt.Sprintf(" %s %8d %s\n", info.SHA512, info.Size, path)
 	}
 
 	releaseFile := indexes.ReleaseFile()
@@ -1000,7 +1043,8 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
 
 // Remove removes published repository, cleaning up directories, files
 func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly.PublishedStorageProvider,
-	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
+	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress,
+	force bool) error {
 	repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return err
@@ -1041,7 +1085,9 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 		err = collection.CleanupPrefixComponentFiles(repo.Prefix, cleanComponents,
 			publishedStorageProvider.GetPublishedStorage(storage), collectionFactory, progress)
 		if err != nil {
-			return err
+			if !force {
+				return fmt.Errorf("cleanup failed, use -force-drop to override: %s", err)
+			}
 		}
 	}
 

deb/publish_test.go

@@ -117,14 +117,19 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.packageCollection.Update(s.p3)
 
 	s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	s.repo.SkipContents = true
 
 	s.repo2, _ = NewPublishedRepo("", "ppa", "maverick", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo2.SkipContents = true
 
 	s.repo3, _ = NewPublishedRepo("", "linux", "natty", nil, []string{"main", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	s.repo3.SkipContents = true
 
 	s.repo4, _ = NewPublishedRepo("", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo4.SkipContents = true
 
 	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo5.SkipContents = true
 
 	poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
 	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
@@ -300,7 +305,7 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	c.Assert(err, IsNil)
 
 	cfr := NewControlFileReader(rf)
-	st, err := cfr.ReadStanza()
+	st, err := cfr.ReadStanza(true)
 	c.Assert(err, IsNil)
 
 	c.Check(st["Origin"], Equals, "ppa squeeze")
@@ -313,13 +318,13 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	cfr = NewControlFileReader(pf)
 
 	for i := 0; i < 3; i++ {
-		st, err = cfr.ReadStanza()
+		st, err = cfr.ReadStanza(false)
 		c.Assert(err, IsNil)
 
 		c.Check(st["Filename"], Equals, "pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
 	}
 
-	st, err = cfr.ReadStanza()
+	st, err = cfr.ReadStanza(false)
 	c.Assert(err, IsNil)
 	c.Assert(st, IsNil)
 
@@ -327,7 +332,7 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	c.Assert(err, IsNil)
 
 	cfr = NewControlFileReader(drf)
-	st, err = cfr.ReadStanza()
+	st, err = cfr.ReadStanza(true)
 	c.Assert(err, IsNil)
 
 	c.Check(st["Archive"], Equals, "squeeze")
@@ -740,7 +745,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
-	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
@@ -760,10 +765,10 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 
-	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
 	c.Check(err, ErrorMatches, ".*not found")
 
-	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil)
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false)
 	c.Check(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
@@ -778,7 +783,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
-	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil)
+	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda")
@@ -800,7 +805,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) {
-	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil)
+	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")

deb/query.go

@@ -7,6 +7,16 @@ import (
 	"strings"
 )
 
+// PackageLike is something like Package :) To be refined later
+type PackageLike interface {
+	GetField(string) string
+	MatchesDependency(Dependency) bool
+	MatchesArchitecture(string) bool
+	GetName() string
+	GetVersion() string
+	GetArchitecture() string
+}
+
 // PackageCatalog is abstraction on top of PackageCollection and PackageList
 type PackageCatalog interface {
 	Scan(q PackageQuery) (result *PackageList)
@@ -18,7 +28,7 @@ type PackageCatalog interface {
 // PackageQuery is interface of predicate on Package
 type PackageQuery interface {
 	// Matches calculates match of condition against package
-	Matches(pkg *Package) bool
+	Matches(pkg PackageLike) bool
 	// Fast returns if search strategy is possible for this query
 	Fast(list PackageCatalog) bool
 	// Query performs search on package list
@@ -63,7 +73,7 @@ type DependencyQuery struct {
 }
 
 // Matches if any of L, R matches
-func (q *OrQuery) Matches(pkg *Package) bool {
+func (q *OrQuery) Matches(pkg PackageLike) bool {
 	return q.L.Matches(pkg) || q.R.Matches(pkg)
 }
 
@@ -89,7 +99,7 @@ func (q *OrQuery) String() string {
 }
 
 // Matches if both of L, R matches
-func (q *AndQuery) Matches(pkg *Package) bool {
+func (q *AndQuery) Matches(pkg PackageLike) bool {
 	return q.L.Matches(pkg) && q.R.Matches(pkg)
 }
 
@@ -120,7 +130,7 @@ func (q *AndQuery) String() string {
 }
 
 // Matches if not matches
-func (q *NotQuery) Matches(pkg *Package) bool {
+func (q *NotQuery) Matches(pkg PackageLike) bool {
 	return !q.Q.Matches(pkg)
 }
 
@@ -141,9 +151,9 @@ func (q *NotQuery) String() string {
 }
 
 // Matches on generic field
-func (q *FieldQuery) Matches(pkg *Package) bool {
+func (q *FieldQuery) Matches(pkg PackageLike) bool {
 	if q.Field == "$Version" {
-		return pkg.MatchesDependency(Dependency{Pkg: pkg.Name, Relation: q.Relation, Version: q.Value, Regexp: q.Regexp})
+		return pkg.MatchesDependency(Dependency{Pkg: pkg.GetName(), Relation: q.Relation, Version: q.Value, Regexp: q.Regexp})
 	}
 	if q.Field == "$Architecture" && q.Relation == VersionEqual {
 		return pkg.MatchesArchitecture(q.Value)
@@ -218,7 +228,7 @@ func (q *FieldQuery) String() string {
 }
 
 // Matches on dependency condition
-func (q *DependencyQuery) Matches(pkg *Package) bool {
+func (q *DependencyQuery) Matches(pkg PackageLike) bool {
 	return pkg.MatchesDependency(q.Dep)
 }
 
@@ -247,8 +257,8 @@ func (q *DependencyQuery) String() string {
 }
 
 // Matches on specific properties
-func (q *PkgQuery) Matches(pkg *Package) bool {
-	return pkg.Name == q.Pkg && pkg.Version == q.Version && pkg.Architecture == q.Arch
+func (q *PkgQuery) Matches(pkg PackageLike) bool {
+	return pkg.GetName() == q.Pkg && pkg.GetVersion() == q.Version && pkg.GetArchitecture() == q.Arch
 }
 
 // Fast is always true for package query

deb/remote.go

@@ -2,12 +2,12 @@ package deb
 
 import (
 	"bytes"
-	"code.google.com/p/go-uuid/uuid"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/http"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
 	"log"
 	"net/url"
@@ -146,7 +146,7 @@ func (repo *RemoteRepo) IsFlat() bool {
 	return repo.Distribution == "" || (strings.HasPrefix(repo.Distribution, ".") && strings.HasSuffix(repo.Distribution, "/"))
 }
 
-// NumPackages return number of packages retrived from remote repo
+// NumPackages return number of packages retrieved from remote repo
 func (repo *RemoteRepo) NumPackages() int {
 	if repo.packageRefs == nil {
 		return 0
@@ -263,7 +263,7 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error
 		}
 		defer inrelease.Close()
 
-		err = verifier.VerifyClearsigned(inrelease)
+		_, err = verifier.VerifyClearsigned(inrelease, true)
 		if err != nil {
 			goto splitsignature
 		}
@@ -304,7 +304,7 @@ ok:
 	defer release.Close()
 
 	sreader := NewControlFileReader(release)
-	stanza, err := sreader.ReadStanza()
+	stanza, err := sreader.ReadStanza(true)
 	if err != nil {
 		return err
 	}
@@ -443,7 +443,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 		sreader := NewControlFileReader(packagesReader)
 
 		for {
-			stanza, err := sreader.ReadStanza()
+			stanza, err := sreader.ReadStanza(false)
 			if err != nil {
 				return err
 			}

deb/remote_test.go

@@ -30,8 +30,8 @@ func (n *NullVerifier) VerifyDetachedSignature(signature, cleartext io.Reader) e
 	return nil
 }
 
-func (n *NullVerifier) VerifyClearsigned(clearsigned io.Reader) error {
-	return nil
+func (n *NullVerifier) VerifyClearsigned(clearsigned io.Reader, hint bool) (*utils.GpgKeyInfo, error) {
+	return nil, nil
 }
 
 func (n *NullVerifier) ExtractClearsigned(clearsigned io.Reader) (text *os.File, err error) {
@@ -43,6 +43,10 @@ func (n *NullVerifier) ExtractClearsigned(clearsigned io.Reader) (text *os.File,
 	return
 }
 
+func (n *NullVerifier) IsClearSigned(clearsign io.Reader) (bool, error) {
+	return false, nil
+}
+
 type PackageListMixinSuite struct {
 	p1, p2, p3 *Package
 	list       *PackageList
@@ -99,7 +103,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) {
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
 	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false)
-	c.Assert(err, ErrorMatches, ".*hexadecimal escape in host.*")
+	c.Assert(err, ErrorMatches, ".*(hexadecimal escape|percent-encoded characters) in host.*")
 }
 
 func (s *RemoteRepoSuite) TestFlatCreation(c *C) {
@@ -330,7 +334,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, false)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -363,7 +367,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, false)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 

deb/snapshot.go

@@ -2,11 +2,11 @@ package deb
 
 import (
 	"bytes"
-	"code.google.com/p/go-uuid/uuid"
 	"errors"
 	"fmt"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
 	"log"
 	"sort"

deb/uploaders.go

@@ -0,0 +1,105 @@
+package deb
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/DisposaBoy/JsonConfigReader"
+	"github.com/smira/aptly/utils"
+	"os"
+)
+
+// UploadersRule is single rule of format: what packages can group or key upload
+type UploadersRule struct {
+	Condition         string       `json:"condition"`
+	Allow             []string     `json:"allow"`
+	Deny              []string     `json:"deny"`
+	CompiledCondition PackageQuery `json:"-" codec:"-"`
+}
+
+func (u UploadersRule) String() string {
+	b, _ := json.Marshal(u)
+	return string(b)
+}
+
+// Uploaders is configuration of restrictions for .changes file importing
+type Uploaders struct {
+	Groups map[string][]string `json:"groups"`
+	Rules  []UploadersRule     `json:"rules"`
+}
+
+func (u *Uploaders) String() string {
+	b, _ := json.Marshal(u)
+	return string(b)
+}
+
+// NewUploadersFromFile loads Uploaders structue from .json file
+func NewUploadersFromFile(path string) (*Uploaders, error) {
+	uploaders := &Uploaders{}
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("error loading uploaders file: %s", err)
+	}
+	defer f.Close()
+
+	err = json.NewDecoder(JsonConfigReader.New(f)).Decode(&uploaders)
+	if err != nil {
+		return nil, fmt.Errorf("error loading uploaders file: %s", err)
+	}
+
+	return uploaders, nil
+}
+
+func (u *Uploaders) expandGroupsInternal(items []string, trail []string) []string {
+	result := []string{}
+
+	for _, item := range items {
+		// stop infinite recursion
+		if utils.StrSliceHasItem(trail, item) {
+			continue
+		}
+
+		group, ok := u.Groups[item]
+		if !ok {
+			result = append(result, item)
+		} else {
+			newTrail := append([]string(nil), trail...)
+			result = append(result, u.expandGroupsInternal(group, append(newTrail, item))...)
+		}
+	}
+
+	return result
+}
+
+// ExpandGroups expands list of keys/groups into list of keys
+func (u *Uploaders) ExpandGroups(items []string) []string {
+	result := u.expandGroupsInternal(items, []string{})
+
+	return utils.StrSliceDeduplicate(result)
+}
+
+// IsAllowed checks whether listed keys are allowed to upload given .changes file
+func (u *Uploaders) IsAllowed(changes *Changes) error {
+	for _, rule := range u.Rules {
+		if rule.CompiledCondition.Matches(changes) {
+			deny := u.ExpandGroups(rule.Deny)
+			for _, key := range changes.SignatureKeys {
+				for _, item := range deny {
+					if item == "*" || key.Matches(utils.GpgKey(item)) {
+						return fmt.Errorf("denied according to rule: %s", rule)
+					}
+				}
+			}
+
+			allow := u.ExpandGroups(rule.Allow)
+			for _, key := range changes.SignatureKeys {
+				for _, item := range allow {
+					if item == "*" || key.Matches(utils.GpgKey(item)) {
+						return nil
+					}
+				}
+			}
+		}
+	}
+
+	return fmt.Errorf("denied as no rule matches")
+}

deb/uploaders_test.go

@@ -0,0 +1,81 @@
+package deb
+
+import (
+	"github.com/smira/aptly/utils"
+	. "gopkg.in/check.v1"
+)
+
+type UploadersSuite struct {
+}
+
+var _ = Suite(&UploadersSuite{})
+
+func (s *UploadersSuite) TestExpandGroups(c *C) {
+	u := &Uploaders{
+		Groups: map[string][]string{
+			"group1": {"key1", "group2"},
+			"group2": {"key1", "key2", "key3", "group3"},
+			"group3": {},
+			"group4": {"key1", "group5"},
+			"group6": {"key1", "group8"},
+			"group7": {"key2", "group6"},
+			"group8": {"group7"},
+		},
+	}
+
+	c.Check(u.ExpandGroups([]string{"group1"}), DeepEquals, []string{"key1", "key2", "key3"})
+	c.Check(u.ExpandGroups([]string{"group2"}), DeepEquals, []string{"key1", "key2", "key3"})
+	c.Check(u.ExpandGroups([]string{"group3"}), DeepEquals, []string{})
+	c.Check(u.ExpandGroups([]string{"group4"}), DeepEquals, []string{"key1", "group5"})
+	c.Check(u.ExpandGroups([]string{"group6"}), DeepEquals, []string{"key1", "key2"})
+	c.Check(u.ExpandGroups([]string{"group7"}), DeepEquals, []string{"key2", "key1"})
+	c.Check(u.ExpandGroups([]string{"group8"}), DeepEquals, []string{"key2", "key1"})
+}
+
+func (s *UploadersSuite) TestIsAllowed(c *C) {
+	u := &Uploaders{
+		Groups: map[string][]string{
+			"group1": {"37E1C17570096AD1", "EC4B033C70096AD1"},
+		},
+		Rules: []UploadersRule{
+			{
+				CompiledCondition: &FieldQuery{Field: "Source", Relation: VersionEqual, Value: "calamares"},
+				Allow:             []string{"*"},
+			},
+			{
+				CompiledCondition: &FieldQuery{Field: "Source", Relation: VersionEqual, Value: "never-calamares"},
+				Deny:              []string{"*"},
+			},
+			{
+				CompiledCondition: &FieldQuery{Field: "Source", Relation: VersionEqual, Value: "some-calamares"},
+				Allow:             []string{"group1", "12345678"},
+			},
+			{
+				CompiledCondition: &FieldQuery{Field: "Source", Relation: VersionEqual, Value: "some-calamares"},
+				Deny:              []string{"45678901", "12345678"},
+			},
+		},
+	}
+
+	// no keys - not allowed
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{}, Stanza: Stanza{"Source": "calamares"}}), ErrorMatches, "denied as no rule matches")
+
+	// no rule - not allowed
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"37E1C17570096AD1", "EC4B033C70096AD1"}, Stanza: Stanza{"Source": "unknown-calamares"}}), ErrorMatches, "denied as no rule matches")
+
+	// first rule: allow anyone do stuff with calamares
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "calamares"}}), IsNil)
+
+	// second rule: nobody is allowed to do stuff with never-calamares
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "never-calamares"}}),
+		ErrorMatches, "denied according to rule: {\"condition\":\"\",\"allow\":null,\"deny\":\\[\"\\*\"\\]}")
+
+	// third rule: anyone from the group or explicit key
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"45678901", "12345678"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"37E1C17570096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"70096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+
+	// fourth rule: some are not allowed
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "45678901"}, Stanza: Stanza{"Source": "some-calamares"}}),
+		ErrorMatches, "denied according to rule: {\"condition\":\"\",\"allow\":null,\"deny\":\\[\"45678901\",\"12345678\"\\]}")
+}

files/package_pool.go

@@ -151,7 +151,7 @@ func (pool *PackagePool) Import(path string, hashMD5 string) error {
 	}
 
 	// create subdirs as necessary
-	err = os.MkdirAll(filepath.Dir(poolPath), 0755)
+	err = os.MkdirAll(filepath.Dir(poolPath), 0777)
 	if err != nil {
 		return err
 	}

files/public.go

@@ -32,7 +32,7 @@ func (storage *PublishedStorage) PublicPath() string {
 
 // MkDir creates directory recursively under public path
 func (storage *PublishedStorage) MkDir(path string) error {
-	return os.MkdirAll(filepath.Join(storage.rootPath, path), 0755)
+	return os.MkdirAll(filepath.Join(storage.rootPath, path), 0777)
 }
 
 // PutFile puts file into published storage at specified path
@@ -87,7 +87,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 	baseName := filepath.Base(sourcePath)
 	poolPath := filepath.Join(storage.rootPath, publishedDirectory)
 
-	err := os.MkdirAll(poolPath, 0755)
+	err := os.MkdirAll(poolPath, 0777)
 	if err != nil {
 		return err
 	}

http/download.go

@@ -1,10 +1,10 @@
 package http
 
 import (
-	"code.google.com/p/mxk/go1/flowcontrol"
 	"compress/bzip2"
 	"compress/gzip"
 	"fmt"
+	"github.com/mxk/go-flowrate/flowrate"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/go-ftp-protocol/protocol"
@@ -75,7 +75,7 @@ func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.
 	}
 
 	if downLimit > 0 {
-		downloader.aggWriter = flowcontrol.NewWriter(progress, downLimit)
+		downloader.aggWriter = flowrate.NewWriter(progress, downLimit)
 	} else {
 		downloader.aggWriter = progress
 	}
@@ -145,6 +145,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
 	}
+	req.Close = true
 
 	proxyURL, _ := downloader.client.Transport.(*http.Transport).Proxy(req)
 	if proxyURL == nil && (req.URL.Scheme == "http" || req.URL.Scheme == "https") {
@@ -166,7 +167,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 		return
 	}
 
-	err = os.MkdirAll(filepath.Dir(task.destination), 0755)
+	err = os.MkdirAll(filepath.Dir(task.destination), 0777)
 	if err != nil {
 		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
@@ -208,6 +209,8 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 			err = fmt.Errorf("%s: sha1 hash mismatch %#v != %#v", task.url, actual.SHA1, task.expected.SHA1)
 		} else if task.expected.SHA256 != "" && actual.SHA256 != task.expected.SHA256 {
 			err = fmt.Errorf("%s: sha256 hash mismatch %#v != %#v", task.url, actual.SHA256, task.expected.SHA256)
+		} else if task.expected.SHA512 != "" && actual.SHA512 != task.expected.SHA512 {
+			err = fmt.Errorf("%s: sha512 hash mismatch %#v != %#v", task.url, actual.SHA512, task.expected.SHA512)
 		}
 
 		if err != nil {
@@ -326,6 +329,10 @@ func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChe
 		}
 
 		if !foundChecksum {
+			if !ignoreMismatch {
+				continue
+			}
+
 			file, err = DownloadTemp(downloader, tryURL)
 		}
 
@@ -344,5 +351,9 @@ func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChe
 
 		return uncompressed, file, err
 	}
+
+	if err == nil {
+		err = fmt.Errorf("no candidates for %s found", url)
+	}
 	return nil, nil, err
 }

http/download_test.go

@@ -257,27 +257,32 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
 	d.ExpectResponse("http://example.com/file.gz", "x")
-	r, file, err = DownloadTryCompression(d, "http://example.com/file", nil, false)
+	r, file, err = DownloadTryCompression(d, "http://example.com/file", nil, true)
 	c.Assert(err, ErrorMatches, "unexpected EOF")
 	c.Assert(d.Empty(), Equals, true)
 }
 
 func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) {
 	d := NewFakeDownloader()
-	_, _, err := DownloadTryCompression(d, "http://example.com/file", nil, false)
+	_, _, err := DownloadTryCompression(d, "http://example.com/file", nil, true)
 	c.Assert(err, ErrorMatches, "unexpected request.*")
 
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
 	d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
 	d.ExpectError("http://example.com/file", errors.New("403"))
-	_, _, err = DownloadTryCompression(d, "http://example.com/file", nil, false)
+	_, _, err = DownloadTryCompression(d, "http://example.com/file", nil, true)
 	c.Assert(err, ErrorMatches, "403")
 
 	d = NewFakeDownloader()
 	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
 	d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
 	d.ExpectResponse("http://example.com/file", rawData)
-	_, _, err = DownloadTryCompression(d, "http://example.com/file", map[string]utils.ChecksumInfo{"file": {Size: 7}}, false)
+	expectedChecksums := map[string]utils.ChecksumInfo{
+		"file.bz2": {Size: 7},
+		"file.gz":  {Size: 7},
+		"file":     {Size: 7},
+	}
+	_, _, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false)
 	c.Assert(err, ErrorMatches, "checksums don't match.*")
 }

man/aptly.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "APTLY" "1" "February 2015" "" ""
+.TH "APTLY" "1" "March 2016" "" ""
 .
 .SH "NAME"
 \fBaptly\fR \- Debian repository management tool
@@ -48,17 +48,22 @@ Configuration file is stored in JSON format (default values shown below):
   "downloadSourcePackages": false,
   "ppaDistributorID": "ubuntu",
   "ppaCodename": "",
+  "skipContentsPublishing": false,
   "S3PublishEndpoints": {
     "test": {
       "region": "us\-east\-1",
       "bucket": "repo",
+      "endpoint": "",
       "awsAccessKeyID": "",
       "awsSecretAccessKey": "",
       "prefix": "",
       "acl": "public\-read",
       "storageClass": "",
       "encryptionMethod": "",
-      "plusWorkaround": false
+      "plusWorkaround": false,
+      "disableMultiDel": false,
+      "forceSigV2": false,
+      "debug": false
     }
   },
   "SwiftPublishEndpoints": {
@@ -133,8 +138,12 @@ specifies paramaters for short PPA url expansion, if left blank they default to
 \fBS3PublishEndpoints\fR
 configuration of Amazon S3 publishing endpoints (see below)
 .
+.TP
+\fBSwiftPublishEndpoints\fR
+configuration of OpenStack Swift publishing endpoints (see below)
+.
 .SH "S3 PUBLISHING ENDPOINTS"
-aptly could be configured to publish repository directly to Amazon S3\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings:
+aptly could be configured to publish repository directly to Amazon S3 (or S3\-compatible cloud storage)\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings:
 .
 .TP
 \fBregion\fR
@@ -145,6 +154,10 @@ Amazon region for S3 bucket (e\.g\. \fBus\-east\-1\fR)
 bucket name
 .
 .TP
+\fBendpoint\fR
+(optional) when using S3\-compatible cloud storage, specify hostname of service endpoint here, region is ignored if endpoint is set (set region to some human\-readable name) (should be left blank for real Amazon S3)
+.
+.TP
 \fBprefix\fR
 (optional) do publishing under specified prefix in the bucket, defaults to no prefix (bucket root)
 .
@@ -166,7 +179,11 @@ bucket name
 .
 .TP
 \fBplusWorkaround\fR
-(optional) workaround misbehavior in apt and Amazon S3 for files with \fB+\fR in filename by creating two copies of package files with \fB+\fR in filename: one original and another one with spaces instead of plus signs With \fBplusWorkaround\fR enabled, package files with plus sign would be stored twice\. aptly might not cleanup files with spaces when published repository is dropped or updated (switched) to new version of repository (snapshot)\.
+(optional) workaround misbehavior in apt and Amazon S3 for files with \fB+\fR in filename by creating two copies of package files with \fB+\fR in filename: one original and another one with spaces instead of plus signs With \fBplusWorkaround\fR enabled, package files with plus sign would be stored twice\. aptly might not cleanup files with spaces when published repository is dropped or updated (switched) to new version of repository (snapshot)
+.
+.TP
+\fBdisableMultiDel\fR
+(optional) for S3\-compatible cloud storages which do not support \fBMultiDel\fR S3 API, enable this setting (file deletion would be slower with this setting enabled)
 .
 .P
 In order to publish to S3, specify endpoint as \fBs3:endpoint\-name:\fR before publishing prefix on the command line, e\.g\.:
@@ -300,6 +317,24 @@ When specified on command line, query may have to be quoted according to shell r
 .P
 \fBaptly repo import percona stable \(cqmysql\-client (>= 3\.6)\(cq\fR
 .
+.SH "PACKAGE DISPLAY FORMAT"
+Some aptly commands (\fBaptly mirror search\fR, \fBaptly package search\fR, \|\.\|\.\|\.) support \fB\-format\fR flag which allows to customize how search results are printed\. Golang templates are used to specify display format, with all package stanza fields available to template\. In addition to package stanza fields aptly provides:
+.
+.TP
+\fBKey\fR
+internal aptly package ID, unique for all packages in aptly (combination of \fBShortKey\fR and \fBFilesHash\fR)\.
+.
+.TP
+\fBFilesHash\fR
+hash that includes MD5 of all packages files\.
+.
+.TP
+\fBShortKey\fR
+package ID, which is unique in single list (mirror, repo, snapshot, \|\.\|\.\|\.), but not unique in whole aptly package collection\.
+.
+.P
+For example, default aptly display format could be presented with the following template: \fB{{\.Package}}_{{\.Version}}_{{\.Architecture}}\fR\. To display package name with dependencies: \fB{{\.Package}} | {{\.Depends}}\fR\. More information on Golang template syntax: http://godoc\.org/text/template
+.
 .SH "GLOBAL OPTIONS"
 .
 .TP
@@ -312,7 +347,7 @@ location of configuration file (default locations are /etc/aptly\.conf, ~/\.aptl
 .
 .TP
 \-\fBdep\-follow\-all\-variants\fR=false
-when processing dependencies, follow a & b if depdency is \(cqa|b\(cq
+when processing dependencies, follow a & b if dependency is \(cqa|b\(cq
 .
 .TP
 \-\fBdep\-follow\-recommends\fR=false
@@ -533,6 +568,10 @@ $ aptly mirror search wheezy\-main \(cq$Architecture (i386), Name (% *\-dev)\(cq
 Options:
 .
 .TP
+\-\fBformat\fR=
+custom format for result printing
+.
+.TP
 \-\fBwith\-deps\fR=false
 include dependencies into search results
 .
@@ -609,6 +648,10 @@ default component when publishing
 \-\fBdistribution\fR=
 default distribution when publishing
 .
+.TP
+\-\fBuploaders\-file\fR=
+uploaders\.json to be used when including \.changes into this repository
+.
 .SH "DELETE LOCAL REPOSITORY"
 \fBaptly\fR \fBrepo\fR \fBdrop\fR \fIname\fR
 .
@@ -655,6 +698,10 @@ default component when publishing
 \-\fBdistribution\fR=
 default distribution when publishing
 .
+.TP
+\-\fBuploaders\-file\fR=
+uploaders\.json to be used when including \.changes into this repository
+.
 .SH "IMPORT PACKAGES FROM MIRROR TO LOCAL REPOSITORY"
 \fBaptly\fR \fBrepo\fR \fBimport\fR \fIsrc\-mirror\fR \fIdst\-repo\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
 .
@@ -790,9 +837,59 @@ $ aptly repo search my\-software \(cq$Architecture (i386), Name (% *\-dev)\(cq
 Options:
 .
 .TP
+\-\fBformat\fR=
+custom format for result printing
+.
+.TP
 \-\fBwith\-deps\fR=false
 include dependencies into search results
 .
+.SH "ADD PACKAGES TO LOCAL REPOSITORIES BASED ON \.CHANGES FILES"
+\fBaptly\fR \fBrepo\fR \fBinclude\fR <file\.changes>|\fIdirectory\fR \fB\|\.\|\.\|\.\fR
+.
+.P
+Command include looks for \.changes files in list of arguments or specified directories\. Each \.changes file is verified, parsed, referenced files are put into separate temporary directory and added into local repository\. Successfully imported files are removed by default\.
+.
+.P
+Additionally uploads could be restricted with <uploaders\.json> file\. Rules in this file control uploads based on GPG key ID of \.changes file signature and queries on \.changes file fields\.
+.
+.P
+Example:
+.
+.P
+$ aptly repo include \-repo=foo\-release incoming/
+.
+.P
+Options:
+.
+.TP
+\-\fBaccept\-unsigned\fR=false
+accept unsigned \.changes files
+.
+.TP
+\-\fBforce\-replace\fR=false
+when adding package that conflicts with existing package, remove existing package
+.
+.TP
+\-\fBignore\-signatures\fR=false
+disable verification of \.changes file signature
+.
+.TP
+\-\fBkeyring\fR=
+gpg keyring to use when verifying Release file (could be specified multiple times)
+.
+.TP
+\-\fBno\-remove\-files\fR=false
+don\(cqt remove files that have been imported successfully into repository
+.
+.TP
+\-\fBrepo\fR={{\.Distribution}}
+which repo should files go to, defaults to Distribution field of \.changes file
+.
+.TP
+\-\fBuploaders\-file\fR=
+path to uploaders\.json file
+.
 .SH "CREATES SNAPSHOT OF MIRROR (LOCAL REPOSITORY) CONTENTS"
 \fBaptly\fR \fBsnapshot\fR \fBcreate\fR \fIname\fR \fBfrom\fR \fBmirror\fR \fImirror\-name\fR \fB|\fR \fBfrom\fR \fBrepo\fR \fIrepo\-name\fR \fB|\fR \fBempty\fR
 .
@@ -1034,6 +1131,10 @@ $ aptly snapshot search wheezy\-main \(cq$Architecture (i386), Name (% *\-dev)\(
 Options:
 .
 .TP
+\-\fBformat\fR=
+custom format for result printing
+.
+.TP
 \-\fBwith\-deps\fR=false
 include dependencies into search results
 .
@@ -1082,6 +1183,13 @@ $ aptly publish drop wheezy
 .
 .IP "" 0
 .
+.P
+Options:
+.
+.TP
+\-\fBforce\-drop\fR=false
+remove published repository even if some files could not be cleaned up
+.
 .SH "LIST OF PUBLISHED REPOSITORIES"
 \fBaptly\fR \fBpublish\fR \fBlist\fR
 .
@@ -1191,6 +1299,10 @@ GPG passhprase\-file for the key (warning: could be insecure)
 GPG secret keyring to use (instead of default)
 .
 .TP
+\-\fBskip\-contents\fR=false
+don\(cqt generate Contents indexes
+.
+.TP
 \-\fBskip\-signing\fR=false
 don\(cqt sign Release files with GPG
 .
@@ -1274,6 +1386,10 @@ GPG passhprase\-file for the key (warning: could be insecure)
 GPG secret keyring to use (instead of default)
 .
 .TP
+\-\fBskip\-contents\fR=false
+don\(cqt generate Contents indexes
+.
+.TP
 \-\fBskip\-signing\fR=false
 don\(cqt sign Release files with GPG
 .
@@ -1348,6 +1464,10 @@ GPG passhprase\-file for the key (warning: could be insecure)
 GPG secret keyring to use (instead of default)
 .
 .TP
+\-\fBskip\-contents\fR=false
+don\(cqt generate Contents indexes
+.
+.TP
 \-\fBskip\-signing\fR=false
 don\(cqt sign Release files with GPG
 .
@@ -1405,6 +1525,10 @@ GPG passhprase\-file for the key (warning: could be insecure)
 GPG secret keyring to use (instead of default)
 .
 .TP
+\-\fBskip\-contents\fR=false
+don\(cqt generate Contents indexes
+.
+.TP
 \-\fBskip\-signing\fR=false
 don\(cqt sign Release files with GPG
 .
@@ -1427,6 +1551,13 @@ $ aptly package search \(cq$Architecture (i386), Name (% *\-dev)\(cq
 .
 .IP "" 0
 .
+.P
+Options:
+.
+.TP
+\-\fBformat\fR=
+custom format for result printing
+.
 .SH "SHOW DETAILS ABOUT PACKAGES MATCHING QUERY"
 \fBaptly\fR \fBpackage\fR \fBshow\fR \fIpackage\-query\fR
 .
@@ -1469,6 +1600,17 @@ Example:
 .P
 $ aptly db cleanup
 .
+.P
+Options:
+.
+.TP
+\-\fBdry\-run\fR=false
+don\(cqt delete anything
+.
+.TP
+\-\fBverbose\fR=false
+be verbose when loading objects/removing them
+.
 .SH "RECOVER DB AFTER CRASH"
 \fBaptly\fR \fBdb\fR \fBrecover\fR
 .
@@ -1500,6 +1642,29 @@ Options:
 \-\fBlisten\fR=:8080
 host:port for HTTP listening
 .
+.SH "START API HTTP SERVICE"
+\fBaptly\fR \fBapi\fR \fBserve\fR
+.
+.P
+Stat HTTP server with aptly REST API\.
+.
+.P
+Example:
+.
+.P
+$ aptly api serve \-listen=:8080
+.
+.P
+Options:
+.
+.TP
+\-\fBlisten\fR=:8080
+host:port for HTTP listening
+.
+.TP
+\-\fBno\-lock\fR=false
+don\(cqt lock the database
+.
 .SH "RENDER GRAPH OF RELATIONSHIPS"
 \fBaptly\fR \fBgraph\fR
 .
@@ -1512,6 +1677,17 @@ Example:
 .P
 $ aptly graph
 .
+.P
+Options:
+.
+.TP
+\-\fBformat\fR=png
+render graph to specified format (png, svg, pdf, etc\.)
+.
+.TP
+\-\fBoutput\fR=
+specify output filename, default is to open result in viewer
+.
 .SH "SHOW CURRENT APTLY\(cqS CONFIG"
 \fBaptly\fR \fBconfig\fR \fBshow\fR
 .
@@ -1555,6 +1731,18 @@ Options:
 \-\fBfilename\fR=
 specifies the filename that contains the commands to run
 .
+.SH "SHOW CURRENT APTLY\(cqS CONFIG"
+\fBaptly\fR \fBconfig\fR \fBshow\fR
+.
+.P
+Command show displays the current aptly configuration\.
+.
+.P
+Example:
+.
+.P
+$ aptly config show
+.
 .SH "ENVIRONMENT"
 If environment variable \fBHTTP_PROXY\fR is set \fBaptly\fR would use its value to proxy all HTTP requests\.
 .
@@ -1574,4 +1762,67 @@ general failure
 command parse failure
 .
 .SH "AUTHORS"
-Andrey Smirnov (me@smira\.ru)
+List of contributors, in chronological order:
+.
+.IP "\[ci]" 4
+Andrey Smirnov (https://github\.com/smira)
+.
+.IP "\[ci]" 4
+Sebastien Binet (https://github\.com/sbinet)
+.
+.IP "\[ci]" 4
+Ryan Uber (https://github\.com/ryanuber)
+.
+.IP "\[ci]" 4
+Simon Aquino (https://github\.com/queeno)
+.
+.IP "\[ci]" 4
+Vincent Batoufflet (https://github\.com/vbatoufflet)
+.
+.IP "\[ci]" 4
+Ivan Kurnosov (https://github\.com/zerkms)
+.
+.IP "\[ci]" 4
+Dmitrii Kashin (https://github\.com/freehck)
+.
+.IP "\[ci]" 4
+Chris Read (https://github\.com/cread)
+.
+.IP "\[ci]" 4
+Rohan Garg (https://github\.com/shadeslayer)
+.
+.IP "\[ci]" 4
+Russ Allbery (https://github\.com/rra)
+.
+.IP "\[ci]" 4
+Sylvain Baubeau (https://github\.com/lebauce)
+.
+.IP "\[ci]" 4
+Andrea Bernardo Ciddio (https://github\.com/bcandrea)
+.
+.IP "\[ci]" 4
+Michael Koval (https://github\.com/mkoval)
+.
+.IP "\[ci]" 4
+Alexander Guy (https://github\.com/alexanderguy)
+.
+.IP "\[ci]" 4
+Sebastien Badia (https://github\.com/sbadia)
+.
+.IP "\[ci]" 4
+Szymon Sobik (https://github\.com/sobczyk)
+.
+.IP "\[ci]" 4
+Paul Krohn (https://github\.com/paul\-krohn)
+.
+.IP "\[ci]" 4
+Vincent Bernat (https://github\.com/vincentbernat)
+.
+.IP "\[ci]" 4
+x539 (https://github\.com/x539)
+.
+.IP "\[ci]" 4
+Phil Frost (https://github\.com/bitglue)
+.
+.IP "" 0
+

man/aptly.1.ronn.tmpl

@@ -40,17 +40,22 @@ Configuration file is stored in JSON format (default values shown below):
       "downloadSourcePackages": false,
       "ppaDistributorID": "ubuntu",
       "ppaCodename": "",
+      "skipContentsPublishing": false,
       "S3PublishEndpoints": {
         "test": {
           "region": "us-east-1",
           "bucket": "repo",
+          "endpoint": "",
           "awsAccessKeyID": "",
           "awsSecretAccessKey": "",
           "prefix": "",
           "acl": "public-read",
           "storageClass": "",
           "encryptionMethod": "",
-          "plusWorkaround": false
+          "plusWorkaround": false,
+          "disableMultiDel": false,
+          "forceSigV2": false,
+          "debug": false
         }
       },
       "SwiftPublishEndpoints": {
@@ -118,7 +123,8 @@ Options:
 
 ## S3 PUBLISHING ENDPOINTS
 
-aptly could be configured to publish repository directly to Amazon S3. First, publishing
+aptly could be configured to publish repository directly to Amazon S3 (or S3-compatible
+cloud storage). First, publishing
 endpoints should be described in aptly configuration file. Each endpoint has name
 and associated settings:
 
@@ -126,6 +132,10 @@ and associated settings:
      Amazon region for S3 bucket (e.g. `us-east-1`)
    * `bucket`:
      bucket name
+   * `endpoint`:
+     (optional) when using S3-compatible cloud storage, specify hostname of service endpoint here,
+     region is ignored if endpoint is set (set region to some human-readable name)
+     (should be left blank for real Amazon S3)
    * `prefix`:
      (optional) do publishing under specified prefix in the bucket, defaults to
      no prefix (bucket root)
@@ -152,7 +162,10 @@ and associated settings:
      and another one with spaces instead of plus signs
      With `plusWorkaround` enabled, package files with plus sign
      would be stored twice. aptly might not cleanup files with spaces when published
-     repository is dropped or updated (switched) to new version of repository (snapshot).
+     repository is dropped or updated (switched) to new version of repository (snapshot)
+   * `disableMultiDel`:
+     (optional) for S3-compatible cloud storages which do not support `MultiDel` S3 API,
+     enable this setting (file deletion would be slower with this setting enabled)
 
 In order to publish to S3, specify endpoint as `s3:endpoint-name:` before
 publishing prefix on the command line, e.g.:
@@ -264,6 +277,27 @@ When specified on command line, query may have to be quoted according to shell r
 
   `aptly repo import percona stable 'mysql-client (>= 3.6)'`
 
+## PACKAGE DISPLAY FORMAT
+
+Some aptly commands (`aptly mirror search`, `aptly package search`, ...) support `-format` flag
+which allows to customize how search results are printed. Golang templates are used to specify
+display format, with all package stanza fields available to template. In addition to package stanza
+fields aptly provides:
+
+ * `Key`:
+   internal aptly package ID, unique for all packages in aptly
+   (combination of `ShortKey` and `FilesHash`).
+
+ * `FilesHash`:
+   hash that includes MD5 of all packages files.
+
+ * `ShortKey`:
+   package ID, which is unique in single list (mirror, repo, snapshot, ...), but not unique
+   in whole aptly package collection.
+
+For example, default aptly display format could be presented with the following template:
+`{{"{{"}}.Package{{"}}"}}_{{"{{"}}.Version{{"}}"}}_{{"{{"}}.Architecture{{"}}"}}`. To display package name with dependencies:
+`{{"{{"}}.Package{{"}}"}} | {{"{{"}}.Depends{{"}}"}}`. More information on Golang template syntax: http://godoc.org/text/template
 
 ## GLOBAL OPTIONS
 
@@ -283,12 +317,16 @@ When specified on command line, query may have to be quoted according to shell r
 
 {{template "command" findCommand . "serve"}}
 
+{{template "command" findCommand . "api"}}
+
 {{template "command" findCommand . "graph"}}
 
 {{template "command" findCommand . "config"}}
 
 {{template "command" findCommand . "task"}}
 
+{{template "command" findCommand . "config"}}
+
 ## ENVIRONMENT
 
 If environment variable `HTTP_PROXY` is set `aptly` would use its value
@@ -309,7 +347,7 @@ to proxy all HTTP requests.
 
 ## AUTHORS
 
-Andrey Smirnov (me@smira.ru)
+{{authors}}
 
 {{end}}
 

man/gen.go

@@ -5,6 +5,7 @@ import (
 	"github.com/smira/aptly/cmd"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -43,6 +44,12 @@ func capitalize(s string) string {
 	return strings.Join(parts, " ")
 }
 
+var authorsS string
+
+func authors() string {
+	return authorsS
+}
+
 func main() {
 	command := cmd.RootCommand()
 	command.UsageLine = "aptly"
@@ -56,9 +63,24 @@ func main() {
 		"findCommand": findCommand,
 		"toUpper":     strings.ToUpper,
 		"capitalize":  capitalize,
+		"authors":     authors,
 	})
 	template.Must(templ.ParseFiles(filepath.Join(filepath.Dir(_File), "aptly.1.ronn.tmpl")))
 
+	authorsF, err := os.Open(filepath.Join(filepath.Dir(_File), "..", "AUTHORS"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsB, err := ioutil.ReadAll(authorsF)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsF.Close()
+
+	authorsS = string(authorsB)
+
 	output, err := os.Create(filepath.Join(filepath.Dir(_File), "aptly.1.ronn"))
 	if err != nil {
 		log.Fatal(err)

s3/public.go

@@ -2,10 +2,15 @@ package s3
 
 import (
 	"fmt"
-	"github.com/mitchellh/goamz/aws"
-	"github.com/mitchellh/goamz/s3"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/corehandlers"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/files"
+	"github.com/smira/go-aws-auth"
 	"os"
 	"path/filepath"
 	"strings"
@@ -14,12 +19,15 @@ import (
 // PublishedStorage abstract file system with published files (actually hosted on S3)
 type PublishedStorage struct {
 	s3               *s3.S3
-	bucket           *s3.Bucket
-	acl              s3.ACL
+	config           *aws.Config
+	bucket           string
+	acl              string
 	prefix           string
 	storageClass     string
 	encryptionMethod string
 	plusWorkaround   bool
+	disableMultiDel  bool
+	pathCache        map[string]string
 }
 
 // Check interface
@@ -28,8 +36,11 @@ var (
 )
 
 // NewPublishedStorageRaw creates published storage from raw aws credentials
-func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL, prefix,
-	storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
+func NewPublishedStorageRaw(
+	bucket, defaultACL, prefix, storageClass, encryptionMethod string,
+	plusWorkaround, disabledMultiDel bool,
+	config *aws.Config,
+) (*PublishedStorage, error) {
 	if defaultACL == "" {
 		defaultACL = "private"
 	}
@@ -38,38 +49,70 @@ func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL
 		storageClass = ""
 	}
 
+	sess := session.New(config)
+
 	result := &PublishedStorage{
-		s3:               s3.New(auth, region),
-		acl:              s3.ACL(defaultACL),
+		s3:               s3.New(sess),
+		bucket:           bucket,
+		config:           config,
+		acl:              defaultACL,
 		prefix:           prefix,
 		storageClass:     storageClass,
 		encryptionMethod: encryptionMethod,
-		plusWorkaround:   plusWorkaround}
-	result.bucket = result.s3.Bucket(bucket)
+		plusWorkaround:   plusWorkaround,
+		disableMultiDel:  disabledMultiDel,
+	}
 
 	return result, nil
 }
 
 // NewPublishedStorage creates new instance of PublishedStorage with specified S3 access
 // keys, region and bucket name
-func NewPublishedStorage(accessKey, secretKey, region, bucket, defaultACL, prefix,
-	storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
-	auth, err := aws.GetAuth(accessKey, secretKey)
-	if err != nil {
-		return nil, err
+func NewPublishedStorage(accessKey, secretKey, sessionToken, region, endpoint, bucket, defaultACL, prefix,
+	storageClass, encryptionMethod string, plusWorkaround, disableMultiDel, forceSigV2, debug bool) (*PublishedStorage, error) {
+
+	config := &aws.Config{
+		Region: aws.String(region),
 	}
 
-	awsRegion, ok := aws.Regions[region]
-	if !ok {
-		return nil, fmt.Errorf("unknown region: %#v", region)
+	if endpoint != "" {
+		config = config.WithEndpoint(endpoint).WithS3ForcePathStyle(true)
 	}
 
-	return NewPublishedStorageRaw(auth, awsRegion, bucket, defaultACL, prefix, storageClass, encryptionMethod, plusWorkaround)
+	if accessKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, sessionToken)
+	}
+
+	if debug {
+		config = config.WithLogLevel(aws.LogDebug)
+	}
+
+	result, err := NewPublishedStorageRaw(bucket, defaultACL, prefix, storageClass,
+		encryptionMethod, plusWorkaround, disableMultiDel, config)
+
+	if err == nil && forceSigV2 {
+		creds := []awsauth.Credentials{}
+
+		if accessKey != "" {
+			creds = append(creds, awsauth.Credentials{
+				AccessKeyID:     accessKey,
+				SecretAccessKey: secretKey,
+			})
+		}
+
+		result.s3.Handlers.Sign.Clear()
+		result.s3.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
+		result.s3.Handlers.Sign.PushBack(func(req *request.Request) {
+			awsauth.SignS3(req.HTTPRequest, creds...)
+		})
+	}
+
+	return result, err
 }
 
 // String
 func (storage *PublishedStorage) String() string {
-	return fmt.Sprintf("S3: %s:%s/%s", storage.s3.Region.Name, storage.bucket.Name, storage.prefix)
+	return fmt.Sprintf("S3: %s:%s/%s", *storage.config.Region, storage.bucket, storage.prefix)
 }
 
 // MkDir creates directory recursively under public path
@@ -83,7 +126,6 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	var (
 		source *os.File
 		err    error
-		fi     os.FileInfo
 	)
 	source, err = os.Open(sourceFilename)
 	if err != nil {
@@ -91,22 +133,20 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	}
 	defer source.Close()
 
-	fi, err = source.Stat()
-	if err != nil {
-		return err
-	}
-
-	headers := map[string][]string{
-		"Content-Type": {"binary/octet-stream"},
+	params := &s3.PutObjectInput{
+		Bucket: aws.String(storage.bucket),
+		Key:    aws.String(filepath.Join(storage.prefix, path)),
+		Body:   source,
+		ACL:    aws.String(storage.acl),
 	}
 	if storage.storageClass != "" {
-		headers["x-amz-storage-class"] = []string{storage.storageClass}
+		params.StorageClass = aws.String(storage.storageClass)
 	}
 	if storage.encryptionMethod != "" {
-		headers["x-amz-server-side-encryption"] = []string{storage.encryptionMethod}
+		params.ServerSideEncryption = aws.String(storage.encryptionMethod)
 	}
 
-	err = storage.bucket.PutReaderHeader(filepath.Join(storage.prefix, path), source, fi.Size(), headers, storage.acl)
+	_, err = storage.s3.PutObject(params)
 	if err != nil {
 		return fmt.Errorf("error uploading %s to %s: %s", sourceFilename, storage, err)
 	}
@@ -119,10 +159,19 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 
 // Remove removes single file under public path
 func (storage *PublishedStorage) Remove(path string) error {
-	err := storage.bucket.Del(filepath.Join(storage.prefix, path))
+	params := &s3.DeleteObjectInput{
+		Bucket: aws.String(storage.bucket),
+		Key:    aws.String(path),
+	}
+	_, err := storage.s3.DeleteObject(params)
 	if err != nil {
 		return fmt.Errorf("error deleting %s from %s: %s", path, storage, err)
 	}
+
+	if storage.plusWorkaround && strings.Index(path, "+") != -1 {
+		// try to remove workaround version, but don't care about result
+		_ = storage.Remove(strings.Replace(path, "+", " ", -1))
+	}
 	return nil
 }
 
@@ -130,32 +179,52 @@ func (storage *PublishedStorage) Remove(path string) error {
 func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
 	const page = 1000
 
-	filelist, err := storage.Filelist(path)
+	filelist, _, err := storage.internalFilelist(path, false)
 	if err != nil {
 		return err
 	}
 
-	numParts := (len(filelist) + page - 1) / page
-
-	for i := 0; i < numParts; i++ {
-		var part []string
-		if i == numParts-1 {
-			part = filelist[i*page:]
-		} else {
-			part = filelist[i*page : (i+1)*page]
+	if storage.disableMultiDel {
+		for i := range filelist {
+			params := &s3.DeleteObjectInput{
+				Bucket: aws.String(storage.bucket),
+				Key:    aws.String(filepath.Join(storage.prefix, path, filelist[i])),
+			}
+			_, err := storage.s3.DeleteObject(params)
+			if err != nil {
+				return fmt.Errorf("error deleting path %s from %s: %s", filelist[i], storage, err)
+			}
 		}
-		paths := make([]string, len(part))
+	} else {
+		numParts := (len(filelist) + page - 1) / page
 
-		for i := range part {
-			paths[i] = filepath.Join(storage.prefix, path, part[i])
-		}
+		for i := 0; i < numParts; i++ {
+			var part []string
+			if i == numParts-1 {
+				part = filelist[i*page:]
+			} else {
+				part = filelist[i*page : (i+1)*page]
+			}
+			paths := make([]*s3.ObjectIdentifier, len(part))
 
-		err = storage.bucket.MultiDel(paths)
-		if err != nil {
-			return fmt.Errorf("error deleting multiple paths from %s: %s", storage, err)
-		}
-		if err != nil {
-			return err
+			for i := range part {
+				paths[i] = &s3.ObjectIdentifier{
+					Key: aws.String(filepath.Join(storage.prefix, path, part[i])),
+				}
+			}
+
+			params := &s3.DeleteObjectsInput{
+				Bucket: aws.String(storage.bucket),
+				Delete: &s3.Delete{
+					Objects: paths,
+					Quiet:   aws.Bool(true),
+				},
+			}
+
+			_, err := storage.s3.DeleteObjects(params)
+			if err != nil {
+				return fmt.Errorf("error deleting multiple paths from %s: %s", storage, err)
+			}
 		}
 	}
 
@@ -179,17 +248,25 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 	poolPath := filepath.Join(storage.prefix, relPath)
 
 	var (
-		dstKey *s3.Key
-		err    error
+		err error
 	)
 
-	dstKey, err = storage.bucket.GetKey(poolPath)
-	if err != nil {
-		if s3err, ok := err.(*s3.Error); !ok || s3err.StatusCode != 404 {
-			return fmt.Errorf("error getting information about %s from %s: %s", poolPath, storage, err)
+	if storage.pathCache == nil {
+		paths, md5s, err := storage.internalFilelist(storage.prefix, true)
+		if err != nil {
+			return fmt.Errorf("error caching paths under prefix: %s", err)
 		}
-	} else {
-		destinationMD5 := strings.Replace(dstKey.ETag, "\"", "", -1)
+
+		storage.pathCache = make(map[string]string, len(paths))
+
+		for i := range paths {
+			storage.pathCache[paths[i]] = md5s[i]
+		}
+	}
+
+	destinationMD5, exists := storage.pathCache[relPath]
+
+	if exists {
 		if destinationMD5 == sourceMD5 {
 			return nil
 		}
@@ -200,51 +277,72 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 		}
 	}
 
-	return storage.PutFile(relPath, sourcePath)
+	err = storage.PutFile(relPath, sourcePath)
+	if err == nil {
+		storage.pathCache[relPath] = sourceMD5
+	}
+
+	return err
 }
 
 // Filelist returns list of files under prefix
 func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
-	result := []string{}
-	marker := ""
+	paths, _, err := storage.internalFilelist(prefix, true)
+	return paths, err
+}
+
+func (storage *PublishedStorage) internalFilelist(prefix string, hidePlusWorkaround bool) (paths []string, md5s []string, err error) {
+	paths = make([]string, 0, 1024)
+	md5s = make([]string, 0, 1024)
 	prefix = filepath.Join(storage.prefix, prefix)
 	if prefix != "" {
 		prefix += "/"
 	}
-	for {
-		contents, err := storage.bucket.List(prefix, "", marker, 1000)
-		if err != nil {
-			return nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
-		}
-		lastKey := ""
-		for _, key := range contents.Contents {
-			if prefix == "" {
-				result = append(result, key.Key)
-			} else {
-				result = append(result, key.Key[len(prefix):])
-			}
-			lastKey = key.Key
-		}
-		if contents.IsTruncated {
-			marker = contents.NextMarker
-			if marker == "" {
-				// From the s3 docs: If response does not include the
-				// NextMarker and it is truncated, you can use the value of the
-				// last Key in the response as the marker in the subsequent
-				// request to get the next set of object keys.
-				marker = lastKey
-			}
-		} else {
-			break
-		}
+
+	params := &s3.ListObjectsInput{
+		Bucket:  aws.String(storage.bucket),
+		Prefix:  aws.String(prefix),
+		MaxKeys: aws.Int64(1000),
 	}
 
-	return result, nil
+	err = storage.s3.ListObjectsPages(params, func(contents *s3.ListObjectsOutput, lastPage bool) bool {
+		for _, key := range contents.Contents {
+			if storage.plusWorkaround && hidePlusWorkaround && strings.Index(*key.Key, " ") != -1 {
+				// if we use plusWorkaround, we want to hide those duplicates
+				/// from listing
+				continue
+			}
+
+			if prefix == "" {
+				paths = append(paths, *key.Key)
+			} else {
+				paths = append(paths, (*key.Key)[len(prefix):])
+			}
+			md5s = append(md5s, strings.Replace(*key.ETag, "\"", "", -1))
+		}
+
+		return true
+	})
+
+	if err != nil {
+		return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
+	}
+
+	return paths, md5s, nil
 }
 
 // RenameFile renames (moves) file
 func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
-	err := storage.bucket.Copy(filepath.Join(storage.prefix, oldName), filepath.Join(storage.prefix, newName), storage.acl)
+	source := fmt.Sprintf("/%s/%s", storage.bucket, filepath.Join(storage.prefix, oldName))
+
+	params := &s3.CopyObjectInput{
+		Bucket:     aws.String(storage.bucket),
+		CopySource: aws.String(source),
+		Key:        aws.String(filepath.Join(storage.prefix, newName)),
+		ACL:        aws.String(storage.acl),
+	}
+
+	_, err := storage.s3.CopyObject(params)
 	if err != nil {
 		return fmt.Errorf("error copying %s -> %s in %s: %s", oldName, newName, storage, err)
 	}

s3/public_test.go

@@ -1,18 +1,21 @@
 package s3
 
 import (
-	"github.com/mitchellh/goamz/aws"
-	"github.com/mitchellh/goamz/s3/s3test"
-	"github.com/smira/aptly/files"
+	"bytes"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
 	. "gopkg.in/check.v1"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+
+	"github.com/smira/aptly/files"
 )
 
 type PublishedStorageSuite struct {
-	srv                      *s3test.Server
+	srv                      *Server
 	storage, prefixedStorage *PublishedStorage
 }
 
@@ -20,18 +23,16 @@ var _ = Suite(&PublishedStorageSuite{})
 
 func (s *PublishedStorageSuite) SetUpTest(c *C) {
 	var err error
-	s.srv, err = s3test.NewServer(&s3test.Config{})
+	s.srv, err = NewServer(&Config{})
 	c.Assert(err, IsNil)
 	c.Assert(s.srv, NotNil)
 
-	auth, _ := aws.GetAuth("aa", "bb")
-	s.storage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "", "", "", false)
+	s.storage, err = NewPublishedStorage("aa", "bb", "", "test-1", s.srv.URL(), "test", "", "", "", "", false, true, false, false)
+	c.Assert(err, IsNil)
+	s.prefixedStorage, err = NewPublishedStorage("aa", "bb", "", "test-1", s.srv.URL(), "test", "", "lala", "", "", false, true, false, false)
 	c.Assert(err, IsNil)
 
-	s.prefixedStorage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "lala", "", "", false)
-	c.Assert(err, IsNil)
-
-	err = s.storage.s3.Bucket("test").PutBucket("private")
+	_, err = s.storage.s3.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String("test")})
 	c.Assert(err, IsNil)
 }
 
@@ -39,10 +40,37 @@ func (s *PublishedStorageSuite) TearDownTest(c *C) {
 	s.srv.Quit()
 }
 
-func (s *PublishedStorageSuite) TestNewPublishedStorage(c *C) {
-	stor, err := NewPublishedStorage("aa", "bbb", "", "", "", "", "", "", false)
-	c.Check(stor, IsNil)
-	c.Check(err, ErrorMatches, "unknown region: .*")
+func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
+	resp, err := s.storage.s3.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(s.storage.bucket),
+		Key:    aws.String(path),
+	})
+	c.Assert(err, IsNil)
+
+	body, err := ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	c.Assert(err, IsNil)
+
+	return body
+}
+
+func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
+	_, err := s.storage.s3.HeadObject(&s3.HeadObjectInput{
+		Bucket: aws.String(s.storage.bucket),
+		Key:    aws.String(path),
+	})
+	c.Assert(err, ErrorMatches, ".*\n.*status code: 404.*")
+}
+
+func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
+	_, err := s.storage.s3.PutObject(&s3.PutObjectInput{
+		Bucket:      aws.String(s.storage.bucket),
+		Key:         aws.String(path),
+		Body:        bytes.NewReader(data),
+		ContentType: aws.String("binary/octet-stream"),
+		ACL:         aws.String("private"),
+	})
+	c.Assert(err, IsNil)
 }
 
 func (s *PublishedStorageSuite) TestPutFile(c *C) {
@@ -53,16 +81,12 @@ func (s *PublishedStorageSuite) TestPutFile(c *C) {
 	err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a"))
 	c.Check(err, IsNil)
 
-	data, err := s.storage.bucket.Get("a/b.txt")
-	c.Check(err, IsNil)
-	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+	c.Check(s.GetFile(c, "a/b.txt"), DeepEquals, []byte("welcome to s3!"))
 
 	err = s.prefixedStorage.PutFile("a/b.txt", filepath.Join(dir, "a"))
 	c.Check(err, IsNil)
 
-	data, err = s.storage.bucket.Get("lala/a/b.txt")
-	c.Check(err, IsNil)
-	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+	c.Check(s.GetFile(c, "lala/a/b.txt"), DeepEquals, []byte("welcome to s3!"))
 }
 
 func (s *PublishedStorageSuite) TestPutFilePlusWorkaround(c *C) {
@@ -75,20 +99,15 @@ func (s *PublishedStorageSuite) TestPutFilePlusWorkaround(c *C) {
 	err = s.storage.PutFile("a/b+c.txt", filepath.Join(dir, "a"))
 	c.Check(err, IsNil)
 
-	data, err := s.storage.bucket.Get("a/b+c.txt")
-	c.Check(err, IsNil)
-	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+	c.Check(s.GetFile(c, "a/b+c.txt"), DeepEquals, []byte("welcome to s3!"))
 
-	data, err = s.storage.bucket.Get("a/b c.txt")
-	c.Check(err, IsNil)
-	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+	c.Check(s.GetFile(c, "a/b c.txt"), DeepEquals, []byte("welcome to s3!"))
 }
 
 func (s *PublishedStorageSuite) TestFilelist(c *C) {
 	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
 	for _, path := range paths {
-		err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")
-		c.Check(err, IsNil)
+		s.PutFile(c, path, []byte("test"))
 	}
 
 	list, err := s.storage.Filelist("")
@@ -108,24 +127,65 @@ func (s *PublishedStorageSuite) TestFilelist(c *C) {
 	c.Check(list, DeepEquals, []string{"a", "b", "c"})
 }
 
-func (s *PublishedStorageSuite) TestRemove(c *C) {
-	err := s.storage.bucket.Put("a/b", []byte("test"), "binary/octet-stream", "private")
+func (s *PublishedStorageSuite) TestFilelistPlusWorkaround(c *C) {
+	s.storage.plusWorkaround = true
+	s.prefixedStorage.plusWorkaround = true
+
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	list, err := s.storage.Filelist("")
 	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a+b", "lala/c", "test/a+1", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a+1"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a+b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestRemove(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b")
+}
+
+func (s *PublishedStorageSuite) TestRemovePlusWorkaround(c *C) {
+	s.storage.plusWorkaround = true
+
+	s.PutFile(c, "a/b+c", []byte("test"))
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.Remove("a/b+c")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b+c")
+	s.AssertNoFile(c, "a/b c")
 
 	err = s.storage.Remove("a/b")
 	c.Check(err, IsNil)
 
-	_, err = s.storage.bucket.Get("a/b")
-	c.Check(err, ErrorMatches, "The specified key does not exist.")
+	s.AssertNoFile(c, "a/b")
 }
 
 func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
-	c.Skip("multiple-delete not available in s3test")
+	s.storage.plusWorkaround = true
 
-	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
 	for _, path := range paths {
-		err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")
-		c.Check(err, IsNil)
+		s.PutFile(c, path, []byte("test"))
 	}
 
 	err := s.storage.RemoveDirs("test", nil)
@@ -133,8 +193,21 @@ func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
 
 	list, err := s.storage.Filelist("")
 	c.Check(err, IsNil)
-	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a+b", "lala/c", "testa"})
+}
 
+func (s *PublishedStorageSuite) TestRemoveDirsPlusWorkaround(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "testa"})
 }
 
 func (s *PublishedStorageSuite) TestRenameFile(c *C) {
@@ -163,31 +236,23 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "c1df1da7a1ce305a3b60af9d5733ac1d", false)
 	c.Check(err, IsNil)
 
-	data, err := s.storage.bucket.Get("pool/main/m/mars-invaders/mars-invaders_1.03.deb")
-	c.Check(err, IsNil)
-	c.Check(data, DeepEquals, []byte("Contents"))
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// duplicate link from pool
 	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "c1df1da7a1ce305a3b60af9d5733ac1d", false)
 	c.Check(err, IsNil)
 
-	data, err = s.storage.bucket.Get("pool/main/m/mars-invaders/mars-invaders_1.03.deb")
-	c.Check(err, IsNil)
-	c.Check(data, DeepEquals, []byte("Contents"))
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with conflict
 	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath2, "e9dfd31cc505d51fc26975250750deab", false)
 	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
 
-	data, err = s.storage.bucket.Get("pool/main/m/mars-invaders/mars-invaders_1.03.deb")
-	c.Check(err, IsNil)
-	c.Check(data, DeepEquals, []byte("Contents"))
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with conflict and force
 	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath2, "e9dfd31cc505d51fc26975250750deab", true)
 	c.Check(err, IsNil)
 
-	data, err = s.storage.bucket.Get("pool/main/m/mars-invaders/mars-invaders_1.03.deb")
-	c.Check(err, IsNil)
-	c.Check(data, DeepEquals, []byte("Spam"))
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
 }

s3/server_test.go

@@ -0,0 +1,699 @@
+package s3
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+const debug = true
+
+type s3Error struct {
+	statusCode int
+	XMLName    struct{} `xml:"Error"`
+	Code       string
+	Message    string
+	BucketName string
+	RequestId  string
+	HostId     string
+}
+
+type action struct {
+	srv   *Server
+	w     http.ResponseWriter
+	req   *http.Request
+	reqId string
+}
+
+// Config controls the internal behaviour of the Server. A nil config is the default
+// and behaves as if all configurations assume their default behaviour. Once passed
+// to NewServer, the configuration must not be modified.
+type Config struct {
+	// Send409Conflict controls how the Server will respond to calls to PUT on a
+	// previously existing bucket. The default is false, and corresponds to the
+	// us-east-1 s3 enpoint. Setting this value to true emulates the behaviour of
+	// all other regions.
+	// http://docs.amazonwebservices.com/AmazonS3/latest/API/ErrorResponses.html
+	Send409Conflict bool
+}
+
+func (c *Config) send409Conflict() bool {
+	if c != nil {
+		return c.Send409Conflict
+	}
+	return false
+}
+
+// Server is a fake S3 server for testing purposes.
+// All of the data for the server is kept in memory.
+type Server struct {
+	url      string
+	reqId    int
+	listener net.Listener
+	mu       sync.Mutex
+	buckets  map[string]*bucket
+	config   *Config
+}
+
+type bucket struct {
+	name    string
+	acl     string
+	ctime   time.Time
+	objects map[string]*object
+}
+
+type object struct {
+	name     string
+	mtime    time.Time
+	meta     http.Header // metadata to return with requests.
+	checksum []byte      // also held as Content-MD5 in meta.
+	data     []byte
+}
+
+// A resource encapsulates the subject of an HTTP request.
+// The resource referred to may or may not exist
+// when the request is made.
+type resource interface {
+	put(a *action) interface{}
+	get(a *action) interface{}
+	post(a *action) interface{}
+	delete(a *action) interface{}
+}
+
+func NewServer(config *Config) (*Server, error) {
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		return nil, fmt.Errorf("cannot listen on localhost: %v", err)
+	}
+	srv := &Server{
+		listener: l,
+		url:      "http://" + l.Addr().String(),
+		buckets:  make(map[string]*bucket),
+		config:   config,
+	}
+	go http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		srv.serveHTTP(w, req)
+	}))
+	return srv, nil
+}
+
+// Quit closes down the server.
+func (srv *Server) Quit() {
+	srv.listener.Close()
+}
+
+// URL returns a URL for the server.
+func (srv *Server) URL() string {
+	return srv.url
+}
+
+func fatalError(code int, codeStr string, errf string, a ...interface{}) {
+	panic(&s3Error{
+		statusCode: code,
+		Code:       codeStr,
+		Message:    fmt.Sprintf(errf, a...),
+	})
+}
+
+// serveHTTP serves the S3 protocol.
+func (srv *Server) serveHTTP(w http.ResponseWriter, req *http.Request) {
+	// ignore error from ParseForm as it's usually spurious.
+	req.ParseForm()
+
+	srv.mu.Lock()
+	defer srv.mu.Unlock()
+
+	if debug {
+		log.Printf("s3test %q %q", req.Method, req.URL)
+	}
+	a := &action{
+		srv:   srv,
+		w:     w,
+		req:   req,
+		reqId: fmt.Sprintf("%09X", srv.reqId),
+	}
+	srv.reqId++
+
+	var r resource
+	defer func() {
+		switch err := recover().(type) {
+		case *s3Error:
+			switch r := r.(type) {
+			case objectResource:
+				err.BucketName = r.bucket.name
+			case bucketResource:
+				err.BucketName = r.name
+			}
+			err.RequestId = a.reqId
+			// TODO HostId
+			w.Header().Set("Content-Type", `xml version="1.0" encoding="UTF-8"`)
+			w.WriteHeader(err.statusCode)
+			xmlMarshal(w, err)
+		case nil:
+		default:
+			panic(err)
+		}
+	}()
+
+	r = srv.resourceForURL(req.URL)
+
+	var resp interface{}
+	switch req.Method {
+	case "PUT":
+		resp = r.put(a)
+	case "GET", "HEAD":
+		resp = r.get(a)
+	case "DELETE":
+		resp = r.delete(a)
+	case "POST":
+		resp = r.post(a)
+	default:
+		fatalError(400, "MethodNotAllowed", "unknown http request method %q", req.Method)
+	}
+	if resp != nil && req.Method != "HEAD" {
+		xmlMarshal(w, resp)
+	}
+}
+
+// xmlMarshal is the same as xml.Marshal except that
+// it panics on error. The marshalling should not fail,
+// but we want to know if it does.
+func xmlMarshal(w io.Writer, x interface{}) {
+	if err := xml.NewEncoder(w).Encode(x); err != nil {
+		panic(fmt.Errorf("error marshalling %#v: %v", x, err))
+	}
+}
+
+// In a fully implemented test server, each of these would have
+// its own resource type.
+var unimplementedBucketResourceNames = map[string]bool{
+	"acl":            true,
+	"lifecycle":      true,
+	"policy":         true,
+	"location":       true,
+	"logging":        true,
+	"notification":   true,
+	"versions":       true,
+	"requestPayment": true,
+	"versioning":     true,
+	"website":        true,
+	"uploads":        true,
+}
+
+var unimplementedObjectResourceNames = map[string]bool{
+	"uploadId": true,
+	"acl":      true,
+	"torrent":  true,
+	"uploads":  true,
+}
+
+var pathRegexp = regexp.MustCompile("/(([^/]+)(/(.*))?)?")
+
+// resourceForURL returns a resource object for the given URL.
+func (srv *Server) resourceForURL(u *url.URL) (r resource) {
+
+	if u.Path == "/" {
+		return serviceResource{
+			buckets: srv.buckets,
+		}
+	}
+
+	m := pathRegexp.FindStringSubmatch(u.Path)
+	if m == nil {
+		fatalError(404, "InvalidURI", "Couldn't parse the specified URI")
+	}
+	bucketName := m[2]
+	objectName := m[4]
+	if bucketName == "" {
+		return nullResource{} // root
+	}
+	b := bucketResource{
+		name:   bucketName,
+		bucket: srv.buckets[bucketName],
+	}
+	q := u.Query()
+	if objectName == "" {
+		for name := range q {
+			if unimplementedBucketResourceNames[name] {
+				return nullResource{}
+			}
+		}
+		return b
+
+	}
+	if b.bucket == nil {
+		fatalError(404, "NoSuchBucket", "The specified bucket does not exist")
+	}
+	objr := objectResource{
+		name:    objectName,
+		version: q.Get("versionId"),
+		bucket:  b.bucket,
+	}
+	for name := range q {
+		if unimplementedObjectResourceNames[name] {
+			return nullResource{}
+		}
+	}
+	if obj := objr.bucket.objects[objr.name]; obj != nil {
+		objr.object = obj
+	}
+	return objr
+}
+
+// nullResource has error stubs for all resource methods.
+type nullResource struct{}
+
+func notAllowed() interface{} {
+	fatalError(400, "MethodNotAllowed", "The specified method is not allowed against this resource")
+	return nil
+}
+
+func (nullResource) put(a *action) interface{}    { return notAllowed() }
+func (nullResource) get(a *action) interface{}    { return notAllowed() }
+func (nullResource) post(a *action) interface{}   { return notAllowed() }
+func (nullResource) delete(a *action) interface{} { return notAllowed() }
+
+const timeFormat = "2006-01-02T15:04:05Z"
+
+type serviceResource struct {
+	buckets map[string]*bucket
+}
+
+func (serviceResource) put(a *action) interface{}    { return notAllowed() }
+func (serviceResource) post(a *action) interface{}   { return notAllowed() }
+func (serviceResource) delete(a *action) interface{} { return notAllowed() }
+
+// GET on an s3 service lists the buckets.
+// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTServiceGET.html
+func (r serviceResource) get(a *action) interface{} {
+	type respBucket struct {
+		Name string
+	}
+
+	type response struct {
+		Buckets []respBucket `xml:">Bucket"`
+	}
+
+	resp := response{}
+
+	for _, bucketPtr := range r.buckets {
+		bkt := respBucket{
+			Name: bucketPtr.name,
+		}
+		resp.Buckets = append(resp.Buckets, bkt)
+	}
+
+	return &resp
+}
+
+type bucketResource struct {
+	name   string
+	bucket *bucket // non-nil if the bucket already exists.
+}
+
+type Owner struct {
+	ID          string
+	DisplayName string
+}
+
+// The ListResp type holds the results of a List bucket operation.
+type ListResp struct {
+	Name       string
+	Prefix     string
+	Delimiter  string
+	Marker     string
+	NextMarker string
+	MaxKeys    int
+	// IsTruncated is true if the results have been truncated because
+	// there are more keys and prefixes than can fit in MaxKeys.
+	// N.B. this is the opposite sense to that documented (incorrectly) in
+	// http://goo.gl/YjQTc
+	IsTruncated    bool
+	Contents       []Key
+	CommonPrefixes []string `xml:">Prefix"`
+}
+
+// The Key type represents an item stored in an S3 bucket.
+type Key struct {
+	Key          string
+	LastModified string
+	Size         int64
+	// ETag gives the hex-encoded MD5 sum of the contents,
+	// surrounded with double-quotes.
+	ETag         string
+	StorageClass string
+	Owner        Owner
+}
+
+// GET on a bucket lists the objects in the bucket.
+// http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTBucketGET.html
+func (r bucketResource) get(a *action) interface{} {
+	if r.bucket == nil {
+		fatalError(404, "NoSuchBucket", "The specified bucket does not exist")
+	}
+	delimiter := a.req.Form.Get("delimiter")
+	marker := a.req.Form.Get("marker")
+	maxKeys := -1
+	if s := a.req.Form.Get("max-keys"); s != "" {
+		i, err := strconv.Atoi(s)
+		if err != nil || i < 0 {
+			fatalError(400, "invalid value for max-keys: %q", s)
+		}
+		maxKeys = i
+	}
+	prefix := a.req.Form.Get("prefix")
+	a.w.Header().Set("Content-Type", "application/xml")
+
+	if a.req.Method == "HEAD" {
+		return nil
+	}
+
+	var objs orderedObjects
+
+	// first get all matching objects and arrange them in alphabetical order.
+	for name, obj := range r.bucket.objects {
+		if strings.HasPrefix(name, prefix) {
+			objs = append(objs, obj)
+		}
+	}
+	sort.Sort(objs)
+
+	if maxKeys <= 0 {
+		maxKeys = 1000
+	}
+	resp := &ListResp{
+		Name:      r.bucket.name,
+		Prefix:    prefix,
+		Delimiter: delimiter,
+		Marker:    marker,
+		MaxKeys:   maxKeys,
+	}
+
+	var prefixes []string
+	for _, obj := range objs {
+		if !strings.HasPrefix(obj.name, prefix) {
+			continue
+		}
+		name := obj.name
+		isPrefix := false
+		if delimiter != "" {
+			if i := strings.Index(obj.name[len(prefix):], delimiter); i >= 0 {
+				name = obj.name[:len(prefix)+i+len(delimiter)]
+				if prefixes != nil && prefixes[len(prefixes)-1] == name {
+					continue
+				}
+				isPrefix = true
+			}
+		}
+		if name <= marker {
+			continue
+		}
+		if len(resp.Contents)+len(prefixes) >= maxKeys {
+			resp.IsTruncated = true
+			break
+		}
+		if isPrefix {
+			prefixes = append(prefixes, name)
+		} else {
+			// Contents contains only keys not found in CommonPrefixes
+			resp.Contents = append(resp.Contents, obj.s3Key())
+		}
+	}
+	resp.CommonPrefixes = prefixes
+	return resp
+}
+
+// orderedObjects holds a slice of objects that can be sorted
+// by name.
+type orderedObjects []*object
+
+func (s orderedObjects) Len() int {
+	return len(s)
+}
+func (s orderedObjects) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+func (s orderedObjects) Less(i, j int) bool {
+	return s[i].name < s[j].name
+}
+
+func (obj *object) s3Key() Key {
+	return Key{
+		Key:          obj.name,
+		LastModified: obj.mtime.Format(timeFormat),
+		Size:         int64(len(obj.data)),
+		ETag:         fmt.Sprintf(`"%x"`, obj.checksum),
+		// TODO StorageClass
+		// TODO Owner
+	}
+}
+
+// DELETE on a bucket deletes the bucket if it's not empty.
+func (r bucketResource) delete(a *action) interface{} {
+	b := r.bucket
+	if b == nil {
+		fatalError(404, "NoSuchBucket", "The specified bucket does not exist")
+	}
+	if len(b.objects) > 0 {
+		fatalError(400, "BucketNotEmpty", "The bucket you tried to delete is not empty")
+	}
+	delete(a.srv.buckets, b.name)
+	return nil
+}
+
+// PUT on a bucket creates the bucket.
+// http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTBucketPUT.html
+func (r bucketResource) put(a *action) interface{} {
+	var created bool
+	if r.bucket == nil {
+		if !validBucketName(r.name) {
+			fatalError(400, "InvalidBucketName", "The specified bucket is not valid")
+		}
+		if loc := locationConstraint(a); loc == "" {
+			fatalError(400, "InvalidRequets", "The unspecified location constraint is incompatible for the region specific endpoint this request was sent to.")
+		}
+		// TODO validate acl
+		r.bucket = &bucket{
+			name: r.name,
+			// TODO default acl
+			objects: make(map[string]*object),
+		}
+		a.srv.buckets[r.name] = r.bucket
+		created = true
+	}
+	if !created && a.srv.config.send409Conflict() {
+		fatalError(409, "BucketAlreadyOwnedByYou", "Your previous request to create the named bucket succeeded and you already own it.")
+	}
+	r.bucket.acl = a.req.Header.Get("x-amz-acl")
+	return nil
+}
+
+func (bucketResource) post(a *action) interface{} {
+	fatalError(400, "Method", "bucket POST method not available")
+	return nil
+}
+
+// validBucketName returns whether name is a valid bucket name.
+// Here are the rules, from:
+// http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/BucketRestrictions.html
+//
+// Can contain lowercase letters, numbers, periods (.), underscores (_),
+// and dashes (-). You can use uppercase letters for buckets only in the
+// US Standard region.
+//
+// Must start with a number or letter
+//
+// Must be between 3 and 255 characters long
+//
+// There's one extra rule (Must not be formatted as an IP address (e.g., 192.168.5.4)
+// but the real S3 server does not seem to check that rule, so we will not
+// check it either.
+//
+func validBucketName(name string) bool {
+	if len(name) < 3 || len(name) > 255 {
+		return false
+	}
+	r := name[0]
+	if !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z') {
+		return false
+	}
+	for _, r := range name {
+		switch {
+		case r >= '0' && r <= '9':
+		case r >= 'a' && r <= 'z':
+		case r == '_' || r == '-':
+		case r == '.':
+		default:
+			return false
+		}
+	}
+	return true
+}
+
+var responseParams = map[string]bool{
+	"content-type":        true,
+	"content-language":    true,
+	"expires":             true,
+	"cache-control":       true,
+	"content-disposition": true,
+	"content-encoding":    true,
+}
+
+type objectResource struct {
+	name    string
+	version string
+	bucket  *bucket // always non-nil.
+	object  *object // may be nil.
+}
+
+// GET on an object gets the contents of the object.
+// http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTObjectGET.html
+func (objr objectResource) get(a *action) interface{} {
+	obj := objr.object
+	if obj == nil {
+		fatalError(404, "NoSuchKey", "The specified key does not exist.")
+	}
+	h := a.w.Header()
+	// add metadata
+	for name, d := range obj.meta {
+		h[name] = d
+	}
+	// override header values in response to request parameters.
+	for name, vals := range a.req.Form {
+		if strings.HasPrefix(name, "response-") {
+			name = name[len("response-"):]
+			if !responseParams[name] {
+				continue
+			}
+			h.Set(name, vals[0])
+		}
+	}
+	if r := a.req.Header.Get("Range"); r != "" {
+		fatalError(400, "NotImplemented", "range unimplemented")
+	}
+	// TODO Last-Modified-Since
+	// TODO If-Modified-Since
+	// TODO If-Unmodified-Since
+	// TODO If-Match
+	// TODO If-None-Match
+	// TODO Connection: close ??
+	// TODO x-amz-request-id
+	h.Set("Content-Length", fmt.Sprint(len(obj.data)))
+	h.Set("ETag", hex.EncodeToString(obj.checksum))
+	h.Set("Last-Modified", obj.mtime.UTC().Format(http.TimeFormat))
+	if a.req.Method == "HEAD" {
+		return nil
+	}
+	// TODO avoid holding the lock when writing data.
+	_, err := a.w.Write(obj.data)
+	if err != nil {
+		// we can't do much except just log the fact.
+		log.Printf("error writing data: %v", err)
+	}
+	return nil
+}
+
+var metaHeaders = map[string]bool{
+	"Content-MD5":         true,
+	"x-amz-acl":           true,
+	"Content-Type":        true,
+	"Content-Encoding":    true,
+	"Content-Disposition": true,
+}
+
+// PUT on an object creates the object.
+func (objr objectResource) put(a *action) interface{} {
+	// TODO Cache-Control header
+	// TODO Expires header
+	// TODO x-amz-server-side-encryption
+	// TODO x-amz-storage-class
+
+	// TODO is this correct, or should we erase all previous metadata?
+	obj := objr.object
+	if obj == nil {
+		obj = &object{
+			name: objr.name,
+			meta: make(http.Header),
+		}
+	}
+
+	var expectHash []byte
+	if c := a.req.Header.Get("Content-MD5"); c != "" {
+		var err error
+		expectHash, err = hex.DecodeString(c)
+		if err != nil || len(expectHash) != md5.Size {
+			fatalError(400, "InvalidDigest", "The Content-MD5 you specified was invalid")
+		}
+	}
+	sum := md5.New()
+	// TODO avoid holding lock while reading data.
+	data, err := ioutil.ReadAll(io.TeeReader(a.req.Body, sum))
+	if err != nil {
+		fatalError(400, "TODO", "read error")
+	}
+	gotHash := sum.Sum(nil)
+	if expectHash != nil && bytes.Compare(gotHash, expectHash) != 0 {
+		fatalError(400, "BadDigest", "The Content-MD5 you specified did not match what we received")
+	}
+	if a.req.ContentLength >= 0 && int64(len(data)) != a.req.ContentLength {
+		fatalError(400, "IncompleteBody", "You did not provide the number of bytes specified by the Content-Length HTTP header")
+	}
+
+	// PUT request has been successful - save data and metadata
+	for key, values := range a.req.Header {
+		key = http.CanonicalHeaderKey(key)
+		if metaHeaders[key] || strings.HasPrefix(key, "X-Amz-Meta-") {
+			obj.meta[key] = values
+		}
+	}
+	obj.data = data
+	obj.checksum = gotHash
+	obj.mtime = time.Now()
+	objr.bucket.objects[objr.name] = obj
+	return nil
+}
+
+func (objr objectResource) delete(a *action) interface{} {
+	delete(objr.bucket.objects, objr.name)
+	return nil
+}
+
+func (objr objectResource) post(a *action) interface{} {
+	fatalError(400, "MethodNotAllowed", "The specified method is not allowed against this resource")
+	return nil
+}
+
+type CreateBucketConfiguration struct {
+	LocationConstraint string
+}
+
+// locationConstraint parses the <CreateBucketConfiguration /> request body (if present).
+// If there is no body, an empty string will be returned.
+func locationConstraint(a *action) string {
+	var body bytes.Buffer
+	if _, err := io.Copy(&body, a.req.Body); err != nil {
+		fatalError(400, "InvalidRequest", err.Error())
+	}
+	if body.Len() == 0 {
+		return ""
+	}
+	var loc CreateBucketConfiguration
+	if err := xml.NewDecoder(&body).Decode(&loc); err != nil {
+		fatalError(400, "InvalidRequest", err.Error())
+	}
+	return loc.LocationConstraint
+}

swift/public_test.go

@@ -1,21 +1,21 @@
 package swift
 
 import (
-	"github.com/ncw/swift/swifttest"
-	"github.com/smira/aptly/files"
-
+	"fmt"
 	. "gopkg.in/check.v1"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"path/filepath"
-)
+	"time"
 
-const (
-	TestAddress = "localhost:5324"
-	AuthURL     = "http://" + TestAddress + "/v1.0"
+	"github.com/ncw/swift/swifttest"
+
+	"github.com/smira/aptly/files"
 )
 
 type PublishedStorageSuite struct {
+	TestAddress, AuthURL     string
 	srv                      *swifttest.SwiftServer
 	storage, prefixedStorage *PublishedStorage
 }
@@ -24,14 +24,20 @@ var _ = Suite(&PublishedStorageSuite{})
 
 func (s *PublishedStorageSuite) SetUpTest(c *C) {
 	var err error
-	s.srv, err = swifttest.NewSwiftServer(TestAddress)
+
+	rand.Seed(int64(time.Now().Nanosecond()))
+
+	s.TestAddress = fmt.Sprintf("localhost:%d", rand.Intn(10000)+20000)
+	s.AuthURL = "http://" + s.TestAddress + "/v1.0"
+
+	s.srv, err = swifttest.NewSwiftServer(s.TestAddress)
 	c.Assert(err, IsNil)
 	c.Assert(s.srv, NotNil)
 
-	s.storage, err = NewPublishedStorage("swifttest", "swifttest", AuthURL, "", "", "test", "")
+	s.storage, err = NewPublishedStorage("swifttest", "swifttest", s.AuthURL, "", "", "test", "")
 	c.Assert(err, IsNil)
 
-	s.prefixedStorage, err = NewPublishedStorage("swifttest", "swifttest", AuthURL, "", "", "test", "lala")
+	s.prefixedStorage, err = NewPublishedStorage("swifttest", "swifttest", s.AuthURL, "", "", "test", "lala")
 	c.Assert(err, IsNil)
 
 	s.storage.conn.ContainerCreate("test", nil)
@@ -42,7 +48,7 @@ func (s *PublishedStorageSuite) TearDownTest(c *C) {
 }
 
 func (s *PublishedStorageSuite) TestNewPublishedStorage(c *C) {
-	stor, err := NewPublishedStorage("swifttest", "swifttest", AuthURL, "", "", "", "")
+	stor, err := NewPublishedStorage("swifttest", "swifttest", s.AuthURL, "", "", "", "")
 	c.Check(stor, NotNil)
 	c.Check(err, IsNil)
 }

system/api_lib.py

@@ -27,7 +27,7 @@ class APITest(BaseTest):
         if APITest.aptly_server is None:
             super(APITest, self).prepare()
 
-            APITest.aptly_server = self._start_process("aptly api serve -listen=%s" % (self.base_url),)
+            APITest.aptly_server = self._start_process("aptly api serve -no-lock -listen=%s" % (self.base_url),)
             time.sleep(1)
 
         if os.path.exists(os.path.join(os.environ["HOME"], ".aptly", "upload")):

system/changes/hardlink_0.2.1.dsc

@@ -0,0 +1,30 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+Format: 1.0
+Source: hardlink
+Binary: hardlink
+Architecture: any
+Version: 0.2.1
+Maintainer: Julian Andres Klode <jak@debian.org>
+Homepage: http://jak-linux.org/projects/hardlink/
+Standards-Version: 3.9.3
+Vcs-Browser: http://git.debian.org/?p=users/jak/hardlink.git;a=summary
+Vcs-Git: git://git.debian.org/git/users/jak/hardlink.git
+Build-Depends: debhelper (>= 9), pkg-config, libpcre3-dev
+Package-List: 
+ hardlink deb utils optional
+Checksums-Sha1: 
+ 6e95b8cba450343ab4dc01902e521f29fbd87ac2 12516 hardlink_0.2.1.tar.gz
+Checksums-Sha256: 
+ 4df0adce005526a1f0e1b38171ddb1f017faae9205f5b1c6dfb0fb4207767271 12516 hardlink_0.2.1.tar.gz
+Files: 
+ 8e2caa4d82f228bac08dc9a38bc6edb3 12516 hardlink_0.2.1.tar.gz
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.12 (GNU/Linux)
+
+iEYEARECAAYFAlUFwywACgkQIdu4nBbbPm2M5wCg0pHD8adE1rY1/DpZ4efRuMXY
+MPMAni4xUtyAnwIvkk3MCE2rFrGP3L78
+=5CU1
+-----END PGP SIGNATURE-----

system/changes/hardlink_0.2.1_amd64.changes

@@ -0,0 +1,39 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+Format: 1.8
+Date: Sat, 12 May 2014 12:57:02 +0200
+Source: hardlink
+Binary: hardlink
+Architecture: source amd64
+Version: 0.2.1
+Distribution: unstable
+Urgency: low
+Maintainer: Julian Andres Klode <jak@debian.org>
+Changed-By: Aptly Tester (don't use it) <test@aptly.info>
+Description: 
+ hardlink   - Hardlinks multiple copies of the same file
+Changes: 
+ hardlink (0.2.1) unstable; urgency=low
+ .
+   * Update just to try it out :)
+Checksums-Sha1: 
+ ff306b8f923653b78e00c45ebbc6c1c734859cdf 949 hardlink_0.2.1.dsc
+ 6e95b8cba450343ab4dc01902e521f29fbd87ac2 12516 hardlink_0.2.1.tar.gz
+ 1ac0e962854dff46f14fa7943746660d3cad1679 12468 hardlink_0.2.1_amd64.deb
+Checksums-Sha256: 
+ c0d7458aa2ca3886cd6885f395a289efbc9a396e6765cbbca45f51fde859ea70 949 hardlink_0.2.1.dsc
+ 4df0adce005526a1f0e1b38171ddb1f017faae9205f5b1c6dfb0fb4207767271 12516 hardlink_0.2.1.tar.gz
+ 668399580590bf1ffcd9eb161b6e574751e15f71820c6e08245dac7c5111a0ee 12468 hardlink_0.2.1_amd64.deb
+Files: 
+ 4efce26825af5842f43961096dd890b3 949 utils optional hardlink_0.2.1.dsc
+ 8e2caa4d82f228bac08dc9a38bc6edb3 12516 utils optional hardlink_0.2.1.tar.gz
+ 2081e20b36c47f82811c25841cc0e41b 12468 utils optional hardlink_0.2.1_amd64.deb
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.12 (GNU/Linux)
+
+iEYEARECAAYFAlUFwywACgkQIdu4nBbbPm1DLACgwW4V8qLQC/QHC/7+t3Iq47Ez
+eesAn3ZYLQvLYRw3wPTKVAPI+AW6Fjxi
+=hRBo
+-----END PGP SIGNATURE-----

system/changes/uploaders1.json

@@ -0,0 +1,7 @@
+// no groups, no rules => deny all
+{
+    "groups": {
+    },
+    "rules": [
+    ]
+}

system/changes/uploaders2.json

@@ -0,0 +1,14 @@
+
+{
+    "groups": {
+        "developers": ["21DBB89C16DB3E6D", "37E1C17570096AD1"],
+    },
+    "rules": [
+        { "condition": "Source (dangerous) | Source (kernel)",
+          "deny": ["*"],
+        },
+        { "condition": "Source (hardlink)",
+          "allow": ["developers", "admins"],
+        }
+    ]
+}

system/changes/uploaders3.json

@@ -0,0 +1 @@
+{

system/changes/uploaders4.json

@@ -0,0 +1,14 @@
+
+{
+    "groups": {
+        "developers": ["21DBB89C16DB3E6D", "37E1C17570096AD1"],
+    },
+    "rules": [
+        { "condition": "Source (dangerous) | Source (kernel)",
+          "deny": ["*"],
+        },
+        { "condition": "Source (hardlink",
+          "allow": ["developers", "admins"],
+        }
+    ]
+}

system/files/flat.key

@@ -1,19 +1,19 @@
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1.4.5 (GNU/Linux)
 
-mQGiBE9p65cRBACFOL5YS4kW6xieXa98meE+RVu1hfBi1n7ajAy+ZQOfNa2Xb9if
-H8WAcwJD4cTZYB19/O/xVxCYf1hnC/T34XGC5PUzMzBDKde86UDqvT4YNHDQA/E1
-I6UUzE0MgLINO4Dt7Mw62koVrlPXklc2Zn83ucZB7YgBzJOIBFUQLghikwCgnubS
-n/9lw8Hm8CIsg4nWtHwHGPED/jXIsH7ON3PBx2wIdRsealsx5sPGHQSlq1grRHcN
-YT5/glXmVqnexY/+IFhcpjjb3vMMQ5LYq8+bDWGVMQx3GZrJs66rwPbo4kZ92OdC
-RTnY/nznJlf5gS86DaFl+NFuO7F1k8ju4CurXXGXPF7nk8cgV6CrYHz1AtNyLVqa
-306IA/9j9rdD/MY9SYT16eFMo7C2ieIS0RxxU3q9w0e8EucQKiHWMtjTPJ0Ik0GO
-TY5lAPasnD6ZBA15XSiTi2Ck2QoZQZCxdtId/nL7lNG4+vQ8HACNDkxxK4yHJiFa
-frMdlWi5cYgAMYzbYPekbhaamDR7Gh4NU7z72QZTPELKyZD/pbRGaG9tZTpEZWVw
-RGl2ZXIxOTc1IE9CUyBQcm9qZWN0IDxob21lOkRlZXBEaXZlcjE5NzVAYnVpbGQu
-b3BlbnN1c2Uub3JnPohlBBMRAgAmBQJPaeuXAhsDBQkEHrAABgsJCAcDAgQVAggD
-BBYCAwECHgECF4AACgkQuCWWGBBIwf+tEgCcDEzTAilZDvOr0OKHHmguuKFXoHMA
-ljX7B6nKOYoiHoGpBeOwr8U5ZB6IRgQTEQIABgUCT2nrlwAKCRA7MBG3a51lI7Rt
-AJ4mYeomQiHHfd+7c8T0JhbGKUIDlACglHyTlouU5vCpUEHDyLvwrHFylpk=
-=b/6f
+mQGiBEeWWZoRBADvY6hFi/SwuwZRLbT2yrW/l+/0zsNXusK/s2K6jjA3i5oDWwG+
+wky3ef+9au2qaxd2spt8iq0x5ph8aI/1YEkdFbHe/dfItcc7NAWl2RTworogkz+R
+4leJNU3tQJNJQYml33LQyNhgQsmMvmZ6xP4NxJZIrWT9nZxiGexb24uTGwCgkOWV
+CpqucQTHnow1ok75gVMRHdkD/3EOtaBAkj5gECBc2WnXk+UzCYBjV1QXmBaLy4sq
+vIq0b3ja+zBJzc2mAviWgNuo2h56dYEsBLaQjkEdJKUEGdfiRkDZQjZEPMrT1HZD
+L2oYzR4xlwSTqR3dtBzXbLhIvfRUNw1CuNVQG8I+r7aBEje7JAVH8X4poGV3oCBh
+faXDA/0a/01gn1d58uSLwDKmBQpUngX/K1U2PchUnJd88DSRqIpzX1Cwk2iu9lJS
+ukE9FV2FNb4heKS2HRXyEz/kEONbSC4G1jSzl0keKrc8RwaX2uTolDGqpyOkXSd+
+WHUcnbzQ5MLahLB3QOocrLc0w6wwPKk2fA5F51mckMUgDtdQ2rQ8aG9tZTptb25r
+ZXlpcSBPQlMgUHJvamVjdCA8aG9tZTptb25rZXlpcUBidWlsZC5vcGVuc3VzZS5v
+cmc+iGYEExECACYFAk/y8O8CGwMFCQx7R1UGCwkIBwMCBBUCCAMEFgIDAQIeAQIX
+gAAKCRCooNd/3/r8Q3MZAJ45GODOQT+bFI8Zjq0C93L7oMxptQCgh/lNR+pYmUcT
+hb1PQ20qsfV5gJuIRgQTEQIABgUCR5ZZmgAKCRA7MBG3a51lI0jzAJsHZxWi1Db3
+J76+37rmZ/2riTo93QCfS5pFjOdqaRPjbfb6bCHLedhhHlM=
+=p7/n
 -----END PGP PUBLIC KEY BLOCK-----

system/lib.py

@@ -154,6 +154,7 @@ class BaseTest(object):
         if not hasattr(command, "__iter__"):
             params = {
                 'files': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files"),
+                'changes': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "changes"),
                 'udebs': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "udebs"),
                 'testfiles': os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), self.__class__.__name__),
                 'aptlyroot': os.path.join(os.environ["HOME"], ".aptly"),
@@ -235,6 +236,8 @@ class BaseTest(object):
             self.verify_match(self.get_gold(gold_name), contents, match_prepare=match_prepare)
         except:
             if self.captureResults:
+                if match_prepare is not None:
+                    contents = match_prepare(contents)
                 with open(self.get_gold_filename(gold_name), "w") as f:
                     f.write(contents)
             else:

system/run.py

@@ -7,6 +7,7 @@ import inspect
 import fnmatch
 import sys
 import traceback
+import random
 
 from lib import BaseTest
 from s3_lib import S3Test
@@ -98,6 +99,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
 
 if __name__ == "__main__":
     os.chdir(os.path.realpath(os.path.dirname(sys.argv[0])))
+    random.seed()
     include_long_tests = False
     capture_results = False
     tests = None

system/s3_lib.py

@@ -22,7 +22,7 @@ class S3Test(BaseTest):
         return super(S3Test, self).fixture_available() and s3_conn is not None
 
     def prepare(self):
-        self.bucket_name = "aptly-sys-test-" + str(uuid.uuid4())
+        self.bucket_name = "aptly-sys-test-" + str(uuid.uuid1())
         self.bucket = s3_conn.create_bucket(self.bucket_name)
         self.configOverride = {"S3PublishEndpoints": {
             "test1": {

system/swift_lib.py

@@ -37,7 +37,7 @@ class SwiftTest(BaseTest):
         return super(SwiftTest, self).fixture_available() and swift_conn is not None
 
     def prepare(self):
-        self.container_name = "aptly-sys-test-" + str(uuid.uuid4())
+        self.container_name = "aptly-sys-test-" + str(uuid.uuid1())
         swift_conn.put_container(self.container_name)
 
         self.configOverride = {"SwiftPublishEndpoints": {

system/t01_version/VersionTest_gold

@@ -1 +1 @@
-aptly version: 0.9.1
+aptly version: 0.9.7

system/t02_config/ConfigShowTest_gold

@@ -12,6 +12,7 @@
     "downloadSourcePackages": false,
     "ppaDistributorID": "ubuntu",
     "ppaCodename": "",
+    "skipContentsPublishing": false,
     "S3PublishEndpoints": {},
     "SwiftPublishEndpoints": {}
 }

system/t02_config/CreateConfigTest_gold

@@ -12,6 +12,7 @@
   "downloadSourcePackages": false,
   "ppaDistributorID": "ubuntu",
   "ppaCodename": "",
+  "skipContentsPublishing": false,
   "S3PublishEndpoints": {},
   "SwiftPublishEndpoints": {}
 }

system/t03_help/MainHelpTest_gold

@@ -13,7 +13,7 @@ package environment to new version.
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t03_help/MainTest_gold

@@ -21,7 +21,7 @@ Use "aptly help <command>" for more information about a command.
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t03_help/MirrorCreateHelpTest_gold

@@ -15,7 +15,7 @@ Example:
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t03_help/MirrorCreateTest_gold

@@ -6,7 +6,7 @@ aptly mirror create - create new mirror
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t03_help/MirrorHelpTest_gold

@@ -17,7 +17,7 @@ Use "mirror help <command>" for more information about a command.
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t03_help/MirrorTest_gold

@@ -17,7 +17,7 @@ Use "mirror help <command>" for more information about a command.
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t03_help/WrongFlagTest_gold

@@ -7,7 +7,7 @@ aptly mirror create - create new mirror
 Options:
   -architectures="": list of architectures to consider during (comma-separated), default to all available
   -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
-  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if dependency is 'a|b'
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests

system/t04_mirror/CreateMirror11Test_gold

@@ -1,10 +1,12 @@
-Downloading http://mirror.yandex.ru/debian/dists/squeeze/InRelease...
-Downloading http://mirror.yandex.ru/debian/dists/squeeze/Release...
-Downloading http://mirror.yandex.ru/debian/dists/squeeze/Release.gpg...
-gpgv:  RSA key ID 473041FA
-gpgv: Good signature from "Debian Archive Automatic Signing Key (6.0/squeeze) <ftpmaster@debian.org>"
-gpgv:  RSA key ID B98321F9
-gpgv: Good signature from "Squeeze Stable Release Key <debian-release@lists.debian.org>"
+Downloading http://mirror.yandex.ru/debian/dists/wheezy/InRelease...
+Downloading http://mirror.yandex.ru/debian/dists/wheezy/Release...
+Downloading http://mirror.yandex.ru/debian/dists/wheezy/Release.gpg...
+gpgv:  RSA key ID 46925553
+gpgv: Good signature from "Debian Archive Automatic Signing Key (7.0/wheezy) <ftpmaster@debian.org>"
+gpgv:  RSA key ID 2B90D010
+gpgv: Good signature from "Debian Archive Automatic Signing Key (8/jessie) <ftpmaster@debian.org>"
+gpgv:  RSA key ID 65FFB764
+gpgv: Good signature from "Wheezy Stable Release Key <debian-release@lists.debian.org>"
 
-Mirror [mirror11]: http://mirror.yandex.ru/debian/ squeeze successfully added.
+Mirror [mirror11]: http://mirror.yandex.ru/debian/ wheezy successfully added.
 You can run 'aptly mirror update mirror11' to download repository contents.

system/t04_mirror/CreateMirror11Test_mirror_show

@@ -1,20 +1,20 @@
 Name: mirror11
 Archive Root URL: http://mirror.yandex.ru/debian/
-Distribution: squeeze
+Distribution: wheezy
 Components: main, contrib, non-free
-Architectures: amd64, armel, i386, ia64, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390, sparc
+Architectures: amd64, armel, armhf, i386, ia64, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390, s390x, sparc
 Download Sources: no
 Download .udebs: no
 Last update: never
 
 Information from release file:
-Architectures: amd64 armel i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 sparc
-Codename: squeeze
+Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
+Codename: wheezy
 Components: main contrib non-free
-Date: Sat, 19 Jul 2014 11:02:06 UTC
-Description:  Debian 6.0.10 Released 19 July 2014
+Date: Sat, 05 Sep 2015 11:44:23 UTC
+Description:  Debian 7.9 Released 05 September 2015
 
 Label: Debian
 Origin: Debian
 Suite: oldstable
-Version: 6.0.10
+Version: 7.9