Version bump: 0.7.1.

Decoding looses value of time.Time field, but that is not critical.

.travis.yml

@@ -1,8 +1,8 @@
 language: go
 
 go:
-  - 1.1
   - 1.2.1
+  - 1.3
   - tip
 
 env:

AUTHORS

@@ -0,0 +1,6 @@
+List of contributors, in chronological order:
+
+* Andrey Smirnov (https://github.com/smira)
+* Sebastien Binet (https://github.com/sbinet)
+* Ryan Uber (https://github.com/ryanuber)
+* Simon Aquino (https://github.com/simonaquino)

Gomfile

@@ -1,12 +1,14 @@
 gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
 gom 'code.google.com/p/go.crypto/ssh/terminal', :commit => '7aa593ce8cea'
-gom 'code.google.com/p/gographviz', :commit => '212766062629'
+gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2'
+gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556'
 gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
 gom 'github.com/cheggaaa/pb', :commit => '74be7a1388046f374ac36e93d46f5d56e856f827'
-gom 'github.com/smira/commander', :commit => '082a3ce267a8225a8ccf94deaf18901223d38fed'
-gom 'github.com/smira/flag', :commit => '0d0aac2addb39050f45e92c5a6252926096dc841'
+gom 'github.com/mitchellh/goamz/s3', :commit => '55f224c07975fddef9d2116600c664e30df3d594'
 gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => 'ff3719c6816e2cd194f05058452d660608e178ac'
+gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
+gom 'github.com/smira/flag', :commit => '0d0aac2addb39050f45e92c5a6252926096dc841'
+gom 'github.com/syndtr/goleveldb/leveldb', :commit => '9888007'
 gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
 gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
 

Makefile

@@ -1,6 +1,6 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-PACKAGES=database deb files http utils
-ALL_PACKAGES=aptly cmd console database deb files http utils
+PACKAGES=database deb files http query s3 utils
+ALL_PACKAGES=aptly cmd console database deb files http query s3 utils
 BINPATH=$(abspath ./_vendor/bin)
 GOM_ENVIRONMENT=-test
 PYTHON?=python
@@ -43,9 +43,7 @@ install:
 	$(GOM) build -o $(BINPATH)/aptly
 
 system-test: install
-ifeq ($(GOVERSION),$(filter $(GOVERSION),go1.2 go1.2.1 devel))
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
-endif
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
 	PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long
 
@@ -69,7 +67,7 @@ package:
 	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
 	gzip root/usr/share/man/man1/aptly.1
 	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
-	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" -C root/ .
+	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 -C root/ .
 	mv aptly_$(VERSION)_*.deb ~
 
 src-package:
@@ -77,9 +75,7 @@ src-package:
 	mkdir -p aptly-$(VERSION)/src/github.com/smira/aptly/
 	cd aptly-$(VERSION)/src/github.com/smira/ && git clone https://github.com/smira/aptly && cd aptly && git checkout v$(VERSION)
 	cd aptly-$(VERSION)/src/github.com/smira/aptly && gom -production install
-	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . -name .git -print | xargs rm -rf
-	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . -name .bzr -print | xargs rm -rf
-	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . -name .hg -print | xargs rm -rf
+	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . \( -name .git -o -name .bzr -o -name .hg \) -print | xargs rm -rf
 	rm -rf aptly-$(VERSION)/src/github.com/smira/aptly/_vendor/{pkg,bin}
 	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
 	rm -rf aptly-$(VERSION)

README.rst

@@ -8,6 +8,10 @@ aptly
 .. image:: https://coveralls.io/repos/smira/aptly/badge.png?branch=HEAD
     :target: https://coveralls.io/r/smira/aptly?branch=HEAD
 
+.. image:: http://gobuild.io/badge/github.com/smira/aptly/download.png
+    :target: http://gobuild.io/github.com/smira/aptly
+
+
 Aptly is a swiss army knife for Debian repository management.
 
 Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support use
@@ -51,7 +55,7 @@ Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
 
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.1+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.2+ required)::
 
     go get -u github.com/mattn/gom
     mkdir -p $GOPATH/src/github.com/smira/aptly

aptly/interfaces.go

@@ -5,7 +5,6 @@ package aptly
 import (
 	"github.com/smira/aptly/utils"
 	"io"
-	"os"
 )
 
 // PackagePool is asbtraction of package pool storage.
@@ -26,26 +25,34 @@ type PackagePool interface {
 
 // PublishedStorage is abstraction of filesystem storing all published repositories
 type PublishedStorage interface {
-	// PublicPath returns root of public part
-	PublicPath() string
 	// MkDir creates directory recursively under public path
 	MkDir(path string) error
-	// CreateFile creates file for writing under public path
-	CreateFile(path string) (*os.File, error)
+	// PutFile puts file into published storage at specified path
+	PutFile(path string, sourceFilename string) error
 	// RemoveDirs removes directory structure under public path
 	RemoveDirs(path string, progress Progress) error
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory string, sourcePool PackagePool, sourcePath string) error
+	LinkFromPool(publishedDirectory string, sourcePool PackagePool, sourcePath, sourceMD5 string, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
-	// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
-	ChecksumsForFile(path string) (utils.ChecksumInfo, error)
 	// RenameFile renames (moves) file
 	RenameFile(oldName, newName string) error
 }
 
+// LocalPublishedStorage is published storage on local filesystem
+type LocalPublishedStorage interface {
+	// PublicPath returns root of public part
+	PublicPath() string
+}
+
+// PublishedStorageProvider is a thing that returns PublishedStorage by name
+type PublishedStorageProvider interface {
+	// GetPublishedStorage returns PublishedStorage by name
+	GetPublishedStorage(name string) PublishedStorage
+}
+
 // Progress is a progress displaying entity, it allows progress bars & simple prints
 type Progress interface {
 	// Writer interface to support progress bar ticking

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
 // Version of aptly
-const Version = "0.5.1"
+const Version = "0.7.1"
 
 // Enable debugging features?
 const EnableDebug = false

cmd/context.go

@@ -8,7 +8,9 @@ import (
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/s3"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"os"
 	"path/filepath"
@@ -27,7 +29,7 @@ type AptlyContext struct {
 	downloader        aptly.Downloader
 	database          database.Storage
 	packagePool       aptly.PackagePool
-	publishedStorage  aptly.PublishedStorage
+	publishedStorages map[string]aptly.PublishedStorage
 	collectionFactory *deb.CollectionFactory
 	dependencyOptions int
 	architecturesList []string
@@ -39,6 +41,9 @@ type AptlyContext struct {
 
 var context *AptlyContext
 
+// Check interface
+var _ aptly.PublishedStorageProvider = &AptlyContext{}
+
 // FatalError is type for panicking to abort execution with non-zero
 // exit code and print meaningful explanation
 type FatalError struct {
@@ -48,7 +53,11 @@ type FatalError struct {
 
 // Fatal panics and aborts execution with exit code 1
 func Fatal(err error) {
-	panic(&FatalError{ReturnCode: 1, Message: err.Error()})
+	returnCode := 1
+	if err == commander.ErrFlagError || err == commander.ErrCommandError {
+		returnCode = 2
+	}
+	panic(&FatalError{ReturnCode: returnCode, Message: err.Error()})
 }
 
 // Config loads and returns current configuration
@@ -138,7 +147,16 @@ func (context *AptlyContext) Progress() aptly.Progress {
 // Downloader returns instance of current downloader
 func (context *AptlyContext) Downloader() aptly.Downloader {
 	if context.downloader == nil {
-		context.downloader = http.NewDownloader(context.Config().DownloadConcurrency, context.Progress())
+		var downloadLimit int64
+		limitFlag := context.flags.Lookup("download-limit")
+		if limitFlag != nil {
+			downloadLimit = limitFlag.Value.Get().(int64)
+		}
+		if downloadLimit == 0 {
+			downloadLimit = context.Config().DownloadLimit
+		}
+		context.downloader = http.NewDownloader(context.Config().DownloadConcurrency,
+			downloadLimit*1024, context.Progress())
 	}
 
 	return context.downloader
@@ -186,12 +204,30 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
 }
 
 // PublishedStorage returns instance of PublishedStorage
-func (context *AptlyContext) PublishedStorage() aptly.PublishedStorage {
-	if context.publishedStorage == nil {
-		context.publishedStorage = files.NewPublishedStorage(context.Config().RootDir)
+func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
+	publishedStorage, ok := context.publishedStorages[name]
+	if !ok {
+		if name == "" {
+			publishedStorage = files.NewPublishedStorage(context.Config().RootDir)
+		} else if strings.HasPrefix(name, "s3:") {
+			params, ok := context.Config().S3PublishRoots[name[3:]]
+			if !ok {
+				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
+			}
+
+			var err error
+			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
+				params.Region, params.Bucket, params.ACL, params.Prefix)
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			Fatal(fmt.Errorf("unknown published storage format: %v", name))
+		}
+		context.publishedStorages[name] = publishedStorage
 	}
 
-	return context.publishedStorage
+	return publishedStorage
 }
 
 // ShutdownContext shuts context down
@@ -227,7 +263,11 @@ func ShutdownContext() {
 func InitContext(flags *flag.FlagSet) error {
 	var err error
 
-	context = &AptlyContext{flags: flags, dependencyOptions: -1}
+	context = &AptlyContext{
+		flags:             flags,
+		dependencyOptions: -1,
+		publishedStorages: map[string]aptly.PublishedStorage{},
+	}
 
 	if aptly.EnableDebug {
 		cpuprofile := flags.Lookup("cpuprofile").Value.String()

cmd/db_cleanup.go

@@ -14,7 +14,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	// collect information about references packages...

cmd/db_recover.go

@@ -11,7 +11,7 @@ func aptlyDbRecover(cmd *commander.Command, args []string) error {
 
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	context.Progress().Printf("Recovering database...\n")

cmd/graph.go

@@ -13,14 +13,15 @@ import (
 	"strings"
 )
 
-func graphvizEscape(s string) string {
-	return fmt.Sprintf("\"%s\"", strings.Replace(s, "\"", "\\\"", 0))
-}
-
 func aptlyGraph(cmd *commander.Command, args []string) error {
 	var err error
 
-	graph := gographviz.NewGraph()
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	graph := gographviz.NewEscape()
 	graph.SetDir(true)
 	graph.SetName("aptly")
 
@@ -34,13 +35,13 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 			return err
 		}
 
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
+		graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "darkgoldenrod1",
-			"label": graphvizEscape(fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
+			"label": fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
 				repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "),
-				strings.Join(repo.Architectures, ", "), repo.NumPackages())),
+				strings.Join(repo.Architectures, ", "), repo.NumPackages()),
 		})
 		existingNodes[repo.UUID] = true
 		return nil
@@ -58,12 +59,12 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 			return err
 		}
 
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
+		graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "mediumseagreen",
-			"label": graphvizEscape(fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
-				repo.Name, repo.Comment, repo.NumPackages())),
+			"label": fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
+				repo.Name, repo.Comment, repo.NumPackages()),
 		})
 		existingNodes[repo.UUID] = true
 		return nil
@@ -91,18 +92,18 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 			description = "Snapshot from repo"
 		}
 
-		graph.AddNode("aptly", graphvizEscape(snapshot.UUID), map[string]string{
+		graph.AddNode("aptly", snapshot.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "cadetblue1",
-			"label":     graphvizEscape(fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages())),
+			"label":     fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages()),
 		})
 
 		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
 			for _, uuid := range snapshot.SourceIDs {
 				_, exists := existingNodes[uuid]
 				if exists {
-					graph.AddEdge(graphvizEscape(uuid), "", graphvizEscape(snapshot.UUID), "", true, nil)
+					graph.AddEdge(uuid, snapshot.UUID, true, nil)
 				}
 			}
 		}
@@ -116,16 +117,19 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 	fmt.Printf("Loading published repos...\n")
 
 	context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
+		graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "darkolivegreen1",
-			"label":     graphvizEscape(fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution, repo.Component, strings.Join(repo.Architectures, ", "))),
+			"label": fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution,
+				strings.Join(repo.Components(), " "), strings.Join(repo.Architectures, ", ")),
 		})
 
-		_, exists := existingNodes[repo.SourceUUID]
-		if exists {
-			graph.AddEdge(graphvizEscape(repo.SourceUUID), "", graphvizEscape(repo.UUID), "", true, nil)
+		for _, uuid := range repo.Sources {
+			_, exists := existingNodes[uuid]
+			if exists {
+				graph.AddEdge(uuid, repo.UUID, true, nil)
+			}
 		}
 
 		return nil

cmd/mirror.go

@@ -54,6 +54,8 @@ func makeCmdMirror() *commander.Command {
 			makeCmdMirrorShow(),
 			makeCmdMirrorDrop(),
 			makeCmdMirrorUpdate(),
+			makeCmdMirrorRename(),
+			makeCmdMirrorEdit(),
 		},
 	}
 }

cmd/mirror_create.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"strings"
@@ -12,7 +13,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	var err error
 	if !(len(args) == 2 && strings.HasPrefix(args[1], "ppa:") || len(args) >= 3) {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	downloadSources := context.Config().DownloadSourcePackages || context.flags.Lookup("with-sources").Value.Get().(bool)
@@ -37,6 +38,16 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
+	repo.Filter = context.flags.Lookup("filter").Value.String()
+	repo.FilterWithDeps = context.flags.Lookup("filter-with-deps").Value.Get().(bool)
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to create mirror: %s", err)
+		}
+	}
+
 	verifier, err := getVerifier(context.flags)
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
@@ -79,6 +90,8 @@ Example:
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/mirror_drop.go

@@ -10,7 +10,7 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]

cmd/mirror_edit.go

@@ -0,0 +1,67 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	context.flags.Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "filter":
+			repo.Filter = flag.Value.String()
+		case "filter-with-deps":
+			repo.FilterWithDeps = flag.Value.Get().(bool)
+		}
+	})
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fmt.Printf("Mirror %s successfully updated.\n", repo)
+	return err
+}
+
+func makeCmdMirrorEdit() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorEdit,
+		UsageLine: "edit <name>",
+		Short:     "edit properties of mirorr",
+		Long: `
+Command edit allows to change settings of mirror:
+filters.
+
+Example:
+
+  $ aptly mirror edit -filter=nginx -filter-with-deps some-mirror
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-edit", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+
+	return cmd
+}

cmd/mirror_list.go

@@ -11,7 +11,7 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)

cmd/mirror_rename.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyMirrorRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.RemoteRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().RemoteRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: mirror %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nMirror %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdMirrorRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames mirror",
+		Long: `
+Command changes name of the mirror.Mirror name should be unique.
+
+Example:
+
+  $ aptly mirror rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/mirror_show.go

@@ -12,7 +12,7 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
@@ -37,6 +37,14 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		downloadSources = "yes"
 	}
 	fmt.Printf("Download Sources: %s\n", downloadSources)
+	if repo.Filter != "" {
+		fmt.Printf("Filter: %s\n", repo.Filter)
+		filterWithDeps := "no"
+		if repo.FilterWithDeps {
+			filterWithDeps = "yes"
+		}
+		fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
+	}
 	if repo.LastDownloadDate.IsZero() {
 		fmt.Printf("Last update: never\n")
 	} else {

cmd/mirror_update.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -10,7 +12,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
@@ -37,7 +39,17 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = repo.Download(context.Progress(), context.Downloader(), context.CollectionFactory(), context.PackagePool(), ignoreMismatch)
+	var filterQuery deb.PackageQuery
+
+	if repo.Filter != "" {
+		filterQuery, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
+
+	err = repo.Download(context.Progress(), context.Downloader(), context.CollectionFactory(), context.PackagePool(), ignoreMismatch,
+		context.DependencyOptions(), filterQuery)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -70,6 +82,7 @@ Example:
 
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/publish.go

@@ -4,6 +4,7 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
+	"strings"
 )
 
 func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
@@ -24,6 +25,20 @@ func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
 
 }
 
+func parsePrefix(param string) (storage, prefix string) {
+	i := strings.LastIndex(param, ":")
+	if i != -1 {
+		storage = param[:i]
+		prefix = param[i+1:]
+		if prefix == "" {
+			prefix = "."
+		}
+	} else {
+		prefix = param
+	}
+	return
+}
+
 func makeCmdPublish() *commander.Command {
 	return &commander.Command{
 		UsageLine: "publish",

cmd/publish_drop.go

@@ -9,17 +9,19 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 1 || len(args) > 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	if len(args) == 2 {
-		prefix = args[1]
+		param = args[1]
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Remove(context.PublishedStorage(), prefix, distribution,
+	storage, prefix := parsePrefix(param)
+
+	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
 		context.CollectionFactory(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
@@ -33,11 +35,11 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 func makeCmdPublishDrop() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishDrop,
-		UsageLine: "drop <distribution> [<prefix>]",
+		UsageLine: "drop <distribution> [[<endpoint>:]<prefix>]",
 		Short:     "remove published repository",
 		Long: `
-Command removes whatever has been published under specified <prefix> and
-<distribution> name.
+Command removes whatever has been published under specified <prefix>,
+publishing <endpoint> and <distribution> name.
 
 Example:
 

cmd/publish_list.go

@@ -11,7 +11,7 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)

cmd/publish_repo.go

@@ -8,13 +8,19 @@ import (
 func makeCmdPublishRepo() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSnapshotOrRepo,
-		UsageLine: "repo <name> [<prefix>]",
+		UsageLine: "repo <name> [[<endpoint>:]<prefix>]",
 		Short:     "publish local repository",
 		Long: `
 Command publishes current state of local repository ready to be consumed
 by apt tools. Published repostiories appear under rootDir/public directory.
 Valid GPG key is required for publishing.
 
+Multiple component repository could be published by specifying several
+components split by commas via -component flag and multiple local
+repositories as the arguments:
+
+    aptly publish repo -component=main,contrib repo-main repo-contrib
+
 It is not recommended to publish local repositories directly unless the
 repository is for testing purposes and changes happen frequently. For
 production usage please take snapshot of repository and publish it
@@ -27,13 +33,14 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-publish-repo", flag.ExitOnError),
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
-	cmd.Flag.String("component", "", "component name to publish")
+	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd
 }

cmd/publish_snapshot.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
@@ -11,61 +12,107 @@ import (
 
 func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	var err error
-	if len(args) < 1 || len(args) > 2 {
+
+	components := strings.Split(context.flags.Lookup("component").Value.String(), ",")
+
+	if len(args) < len(components) || len(args) > len(components)+1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	name := args[0]
-
-	var prefix string
-	if len(args) == 2 {
-		prefix = args[1]
+	var param string
+	if len(args) == len(components)+1 {
+		param = args[len(components)]
+		args = args[0 : len(args)-1]
 	} else {
-		prefix = ""
+		param = ""
 	}
+	storage, prefix := parsePrefix(param)
 
 	var (
-		source  interface{}
+		sources = []interface{}{}
 		message string
 	)
 
 	if cmd.Name() == "snapshot" {
-		var snapshot *deb.Snapshot
-		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
-		if err != nil {
-			return fmt.Errorf("unable to publish: %s", err)
+		var (
+			snapshot     *deb.Snapshot
+			emptyWarning = false
+			parts        = []string{}
+		)
+
+		for _, name := range args {
+			snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			sources = append(sources, snapshot)
+			parts = append(parts, snapshot.Name)
+
+			if snapshot.NumPackages() == 0 {
+				emptyWarning = true
+			}
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
-			return fmt.Errorf("unable to publish: %s", err)
+		if len(parts) == 1 {
+			message = fmt.Sprintf("Snapshot %s has", parts[0])
+		} else {
+			message = fmt.Sprintf("Snapshots %s have", strings.Join(parts, ", "))
+
 		}
 
-		source = snapshot
-		message = fmt.Sprintf("Snapshot %s", snapshot.Name)
+		if emptyWarning {
+			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
+		}
 	} else if cmd.Name() == "repo" {
-		var localRepo *deb.LocalRepo
-		localRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
-		if err != nil {
-			return fmt.Errorf("unable to publish: %s", err)
+		var (
+			localRepo    *deb.LocalRepo
+			emptyWarning = false
+			parts        = []string{}
+		)
+
+		for _, name := range args {
+			localRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			err = context.CollectionFactory().LocalRepoCollection().LoadComplete(localRepo)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			sources = append(sources, localRepo)
+			parts = append(parts, localRepo.Name)
+
+			if localRepo.NumPackages() == 0 {
+				emptyWarning = true
+			}
 		}
 
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(localRepo)
-		if err != nil {
-			return fmt.Errorf("unable to publish: %s", err)
+		if len(parts) == 1 {
+			message = fmt.Sprintf("Local repo %s has", parts[0])
+		} else {
+			message = fmt.Sprintf("Local repos %s have", strings.Join(parts, ", "))
+
 		}
 
-		source = localRepo
-		message = fmt.Sprintf("Local repo %s", localRepo.Name)
+		if emptyWarning {
+			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
+		}
 	} else {
 		panic("unknown command")
 	}
 
-	component := context.flags.Lookup("component").Value.String()
 	distribution := context.flags.Lookup("distribution").Value.String()
 
-	published, err := deb.NewPublishedRepo(prefix, distribution, component, context.ArchitecturesList(), source, context.CollectionFactory())
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -83,7 +130,13 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context.PublishedStorage(), context.CollectionFactory(), signer, context.Progress())
+	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -93,19 +146,25 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	prefix, component, distribution = published.Prefix, published.Component, published.Distribution
+	var repoComponents string
+	prefix, repoComponents, distribution = published.Prefix, strings.Join(published.Components(), " "), published.Distribution
 	if prefix == "." {
 		prefix = ""
 	} else if !strings.HasSuffix(prefix, "/") {
 		prefix += "/"
 	}
 
-	context.Progress().Printf("\n%s has been successfully published.\nPlease setup your webserver to serve directory '%s' with autoindexing.\n",
-		message, context.PublishedStorage().PublicPath())
+	context.Progress().Printf("\n%s been successfully published.\n", message)
+
+	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.LocalPublishedStorage); ok {
+		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
+			localStorage.PublicPath())
+	}
+
 	context.Progress().Printf("Now you can add following line to apt sources:\n")
-	context.Progress().Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, component)
+	context.Progress().Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
 	if utils.StrSliceHasItem(published.Architectures, "source") {
-		context.Progress().Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, component)
+		context.Progress().Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
 	}
 	context.Progress().Printf("Don't forget to add your GPG key to apt with apt-key.\n")
 	context.Progress().Printf("\nYou can also use `aptly serve` to publish your repositories over HTTP quickly.\n")
@@ -116,13 +175,19 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 func makeCmdPublishSnapshot() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSnapshotOrRepo,
-		UsageLine: "snapshot <name> [<prefix>]",
+		UsageLine: "snapshot <name> [[<endpoint>:]<prefix>]",
 		Short:     "publish snapshot",
 		Long: `
 Command publishes snapshot as Debian repository ready to be consumed
 by apt tools. Published repostiories appear under rootDir/public directory.
 Valid GPG key is required for publishing.
 
+Multiple component repository could be published by specifying several
+components split by commas via -component flag and multiple snapshots
+as the arguments:
+
+    aptly publish snapshot -component=main,contrib snap-main snap-contrib
+
 Example:
 
     $ aptly publish snapshot wheezy-main
@@ -130,13 +195,14 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-publish-snapshot", flag.ExitOnError),
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
-	cmd.Flag.String("component", "", "component name to publish")
+	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd
 }

cmd/publish_switch.go

@@ -5,43 +5,39 @@ import (
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
+	"strings"
 )
 
 func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	var err error
-	if len(args) < 2 || len(args) > 3 {
+
+	components := strings.Split(context.flags.Lookup("component").Value.String(), ",")
+
+	if len(args) < len(components)+1 || len(args) > len(components)+2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	var (
-		name     string
+		names    []string
 		snapshot *deb.Snapshot
 	)
 
-	if len(args) == 3 {
-		prefix = args[1]
-		name = args[2]
+	if len(args) == len(components)+2 {
+		param = args[1]
+		names = args[2:]
 	} else {
-		name = args[1]
+		names = args[1:]
 	}
 
-	snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
-	if err != nil {
-		return fmt.Errorf("unable to switch: %s", err)
-	}
-
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-	if err != nil {
-		return fmt.Errorf("unable to switch: %s", err)
-	}
+	storage, prefix := parsePrefix(param)
 
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByPrefixDistribution(prefix, distribution)
+	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -55,14 +51,41 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	published.UpdateSnapshot(snapshot)
+	publishedComponents := published.Components()
+	if len(components) == 1 && len(publishedComponents) == 1 && components[0] == "" {
+		components = publishedComponents
+	}
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names))
+	}
+
+	for i, component := range components {
+		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(names[i])
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
+
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
+
+		published.UpdateSnapshot(component, snapshot)
+	}
 
 	signer, err := getSigner(context.flags)
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context.PublishedStorage(), context.CollectionFactory(), signer, context.Progress())
+	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -72,8 +95,8 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, published.Component,
-		context.PublishedStorage(), context.CollectionFactory(), context.Progress())
+	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -86,11 +109,18 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 func makeCmdPublishSwitch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSwitch,
-		UsageLine: "switch <distribution> [<prefix>] <new-snapshot>",
+		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
 		Short:     "update published repository by switching to new snapshot",
 		Long: `
 Command switches in-place published repository with new snapshot contents. All
-publishing parameters are preserved (architecture list, distribution, component).
+publishing parameters are preserved (architecture list, distribution,
+component).
+
+For multiple component repositories, flag -component should be given with
+list of components to update. Corresponding snapshots should be given in the
+same order, e.g.:
+
+	aptly publish update -component=main,contrib wheezy wh-main wh-contrib
 
 Example:
 
@@ -102,6 +132,8 @@ Example:
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd
 }

cmd/publish_update.go

@@ -11,19 +11,20 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 1 || len(args) > 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	if len(args) == 2 {
-		prefix = args[1]
+		param = args[1]
 	}
+	storage, prefix := parsePrefix(param)
 
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByPrefixDistribution(prefix, distribution)
+	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -37,14 +38,23 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	published.UpdateLocalRepo()
+	components := published.Components()
+	for _, component := range components {
+		published.UpdateLocalRepo(component)
+	}
 
 	signer, err := getSigner(context.flags)
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context.PublishedStorage(), context.CollectionFactory(), signer, context.Progress())
+	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -54,8 +64,8 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, published.Component,
-		context.PublishedStorage(), context.CollectionFactory(), context.Progress())
+	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -68,7 +78,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 func makeCmdPublishUpdate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishUpdate,
-		UsageLine: "update <distribution> [<prefix>]",
+		UsageLine: "update <distribution> [[<endpoint>:]<prefix>]",
 		Short:     "update published local repository",
 		Long: `
 Command re-publishes (updates) published local repository. <distribution>
@@ -76,6 +86,9 @@ and <prefix> should be occupied with local repository published
 using command aptly publish repo. Update happens in-place with
 minimum possible downtime for published repository.
 
+For multiple component published repositories, all local repositories
+are updated.
+
 Example:
 
     $ aptly publish update wheezy ppa
@@ -86,6 +99,7 @@ Example:
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd
 }

cmd/repo.go

@@ -19,6 +19,7 @@ func makeCmdRepo() *commander.Command {
 			makeCmdRepoMove(),
 			makeCmdRepoRemove(),
 			makeCmdRepoShow(),
+			makeCmdRepoRename(),
 		},
 	}
 }

cmd/repo_add.go

@@ -16,7 +16,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
@@ -41,11 +41,13 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 	}
 
 	packageFiles := []string{}
+	failedFiles := []string{}
 
 	for _, location := range args[1:] {
 		info, err2 := os.Stat(location)
 		if err2 != nil {
 			context.Progress().ColoredPrintf("@y[!]@| @!Unable to process %s: %s@|", location, err2)
+			failedFiles = append(failedFiles, location)
 			continue
 		}
 		if info.IsDir() {
@@ -68,6 +70,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 				packageFiles = append(packageFiles, location)
 			} else {
 				context.Progress().ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
+				failedFiles = append(failedFiles, location)
 				continue
 			}
 		}
@@ -100,6 +103,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		}
 		if err != nil {
 			context.Progress().ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
+			failedFiles = append(failedFiles, file)
 			continue
 		}
 
@@ -118,6 +122,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		err = context.PackagePool().Import(file, checksums.MD5)
 		if err != nil {
 			context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", file, err)
+			failedFiles = append(failedFiles, file)
 			continue
 		}
 
@@ -132,6 +137,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 			err = context.PackagePool().Import(sourceFile, f.Checksums.MD5)
 			if err != nil {
 				context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", sourceFile, err)
+				failedFiles = append(failedFiles, file)
 				break
 			}
 
@@ -145,12 +151,14 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		err = context.CollectionFactory().PackageCollection().Update(p)
 		if err != nil {
 			context.Progress().ColoredPrintf("@y[!]@| @!Unable to save package %s: %s@|", p, err)
+			failedFiles = append(failedFiles, file)
 			continue
 		}
 
 		err = list.Add(p)
 		if err != nil {
 			context.Progress().ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
+			failedFiles = append(failedFiles, file)
 			continue
 		}
 
@@ -176,6 +184,15 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		}
 	}
 
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
 	return err
 }
 

cmd/repo_copy.go

@@ -8,10 +8,10 @@ import (
 func makeCmdRepoCopy() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "copy <src-name> <dst-name> <package-spec> ...",
+		UsageLine: "copy <src-name> <dst-name> <package-query> ...",
 		Short:     "copy packages between local repositories",
 		Long: `
-Command copy copies packages matching <package-spec> from local repo
+Command copy copies packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
 Example:

cmd/repo_create.go

@@ -11,7 +11,7 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	repo := deb.NewLocalRepo(args[0], context.flags.Lookup("comment").Value.String())

cmd/repo_drop.go

@@ -10,7 +10,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]

cmd/repo_edit.go

@@ -10,7 +10,7 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[0])

cmd/repo_import.go

@@ -8,10 +8,10 @@ import (
 func makeCmdRepoImport() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "import <src-mirror> <dst-repo> <package-spec> ...",
+		UsageLine: "import <src-mirror> <dst-repo> <package-query> ...",
 		Short:     "import packages from mirror to local repository",
 		Long: `
-Command import looks up packages matching <package-spec> in mirror <src-mirror>
+Command import looks up packages matching <package-query> in mirror <src-mirror>
 and copies them to local repo <dst-repo>.
 
 Example:

cmd/repo_list.go

@@ -11,7 +11,7 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)

cmd/repo_move.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"sort"
@@ -12,7 +13,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 3 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	command := cmd.Name()
@@ -105,7 +106,15 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	toProcess, err := srcList.Filter(args[2:], withDeps, dstList, context.DependencyOptions(), architecturesList)
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i := 0; i < len(args)-2; i++ {
+		queries[i], err = query.Parse(args[i+2])
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+	}
+
+	toProcess, err := srcList.Filter(queries, withDeps, dstList, context.DependencyOptions(), architecturesList)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
@@ -162,10 +171,10 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 func makeCmdRepoMove() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "move <src-name> <dst-name> <package-spec> ...",
+		UsageLine: "move <src-name> <dst-name> <package-query> ...",
 		Short:     "move packages between local repositories",
 		Long: `
-Command move moves packages matching <package-spec> from local repo
+Command move moves packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
 Example:

cmd/repo_remove.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -11,7 +12,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
@@ -33,8 +34,16 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
+	queries := make([]deb.PackageQuery, len(args)-1)
+	for i := 0; i < len(args)-1; i++ {
+		queries[i], err = query.Parse(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to remove: %s", err)
+		}
+	}
+
 	list.PrepareIndex()
-	toRemove, err := list.Filter(args[1:], false, nil, 0, nil)
+	toRemove, err := list.Filter(queries, false, nil, 0, nil)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -62,10 +71,10 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 func makeCmdRepoRemove() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoRemove,
-		UsageLine: "remove <name> <package-spec> ...",
+		UsageLine: "remove <name> <package-query> ...",
 		Short:     "remove packages from local repository",
 		Long: `
-Commands removes packages matching <package-spec> from local repository
+Commands removes packages matching <package-query> from local repository
 <name>. If removed packages are not referenced by other repos or
 snapshots, they can be removed completely (including files) by running
 'aptly db cleanup'.

cmd/repo_rename.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyRepoRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.LocalRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	repo, err = context.CollectionFactory().LocalRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().LocalRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: local repo %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nLocal repo %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdRepoRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames local repository",
+		Long: `
+Command changes name of the local repo. Local repo name should be unique.
+
+Example:
+
+  $ aptly repo rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/repo_show.go

@@ -10,7 +10,7 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]

cmd/serve.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
@@ -10,11 +11,17 @@ import (
 	"net/http"
 	"os"
 	"sort"
+	"strings"
 )
 
 func aptlyServe(cmd *commander.Command, args []string) error {
 	var err error
 
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
 	if context.CollectionFactory().PublishedRepoCollection().Len() == 0 {
 		fmt.Printf("No published repositories, unable to serve.\n")
 		return nil
@@ -69,15 +76,15 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		}
 
 		fmt.Printf("# %s\ndeb http://%s:%s/%s %s %s\n",
-			repo, listenHost, listenPort, prefix, repo.Distribution, repo.Component)
+			repo, listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 
 		if utils.StrSliceHasItem(repo.Architectures, "source") {
 			fmt.Printf("deb-src http://%s:%s/%s %s %s\n",
-				listenHost, listenPort, prefix, repo.Distribution, repo.Component)
+				listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 		}
 	}
 
-	publicPath := context.PublishedStorage().PublicPath()
+	publicPath := context.GetPublishedStorage("").(aptly.LocalPublishedStorage).PublicPath()
 	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)

cmd/snapshot.go

@@ -17,6 +17,7 @@ func makeCmdSnapshot() *commander.Command {
 			makeCmdSnapshotDiff(),
 			makeCmdSnapshotMerge(),
 			makeCmdSnapshotDrop(),
+			makeCmdSnapshotRename(),
 		},
 	}
 }

cmd/snapshot_create.go

@@ -61,7 +61,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		snapshot = deb.NewSnapshotFromPackageList(snapshotName, nil, packageList, "Created as empty")
 	} else {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	err = context.CollectionFactory().SnapshotCollection().Add(snapshot)

cmd/snapshot_diff.go

@@ -10,7 +10,7 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	onlyMatching := context.flags.Lookup("only-matching").Value.Get().(bool)

cmd/snapshot_drop.go

@@ -10,7 +10,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]

cmd/snapshot_list.go

@@ -7,40 +7,83 @@ import (
 	"sort"
 )
 
+// Snapshot sorting methods
+const (
+	SortName = iota
+	SortTime
+)
+
+type snapshotListToSort struct {
+	list       []*deb.Snapshot
+	sortMethod int
+}
+
+func parseSortMethod(sortMethod string) (int, error) {
+	switch sortMethod {
+	case "time", "Time":
+		return SortTime, nil
+	case "name", "Name":
+		return SortName, nil
+	}
+
+	return -1, fmt.Errorf("sorting method \"%s\" unknown", sortMethod)
+}
+
+func (s snapshotListToSort) Swap(i, j int) {
+	s.list[i], s.list[j] = s.list[j], s.list[i]
+}
+
+func (s snapshotListToSort) Less(i, j int) bool {
+	switch s.sortMethod {
+	case SortName:
+		return s.list[i].Name < s.list[j].Name
+	case SortTime:
+		return s.list[i].CreatedAt.Before(s.list[j].CreatedAt)
+	}
+	panic("unknown sort method")
+}
+
+func (s snapshotListToSort) Len() int {
+	return len(s.list)
+}
+
 func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
 
-	snapshots := make([]string, context.CollectionFactory().SnapshotCollection().Len())
+	snapshotsToSort := &snapshotListToSort{}
+	snapshotsToSort.list = make([]*deb.Snapshot, context.CollectionFactory().SnapshotCollection().Len())
+	snapshotsToSort.sortMethod, err = parseSortMethod(sortMethodString)
+	if err != nil {
+		return err
+	}
 
 	i := 0
 	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		if raw {
-			snapshots[i] = snapshot.Name
-		} else {
-			snapshots[i] = snapshot.String()
-		}
+		snapshotsToSort.list[i] = snapshot
 		i++
+
 		return nil
 	})
 
-	sort.Strings(snapshots)
+	sort.Sort(snapshotsToSort)
 
 	if raw {
-		for _, snapshot := range snapshots {
-			fmt.Printf("%s\n", snapshot)
+		for _, snapshot := range snapshotsToSort.list {
+			fmt.Printf("%s\n", snapshot.Name)
 		}
 	} else {
-		if len(snapshots) > 0 {
+		if len(snapshotsToSort.list) > 0 {
 			fmt.Printf("List of snapshots:\n")
 
-			for _, snapshot := range snapshots {
-				fmt.Printf(" * %s\n", snapshot)
+			for _, snapshot := range snapshotsToSort.list {
+				fmt.Printf(" * %s\n", snapshot.String())
 			}
 
 			fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
@@ -67,6 +110,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+	cmd.Flag.String("sort", "name", "display list in 'name' or creation 'time' order")
 
 	return cmd
 }

cmd/snapshot_merge.go

@@ -11,7 +11,7 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	sources := make([]*deb.Snapshot, len(args)-1)
@@ -29,10 +29,15 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	}
 
 	latest := context.flags.Lookup("latest").Value.Get().(bool)
-	overrideMatching := !latest
+    noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
+
+    if noRemove && latest {
+        return fmt.Errorf("-no-remove and -latest can't be specified together")
+    }
+
+	overrideMatching := !latest && !noRemove
 
 	result := sources[0].RefList()
-
 	for i := 1; i < len(sources); i++ {
 		result = result.Merge(sources[i].RefList(), overrideMatching)
 	}
@@ -79,6 +84,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("latest", false, "use only the latest version of each package")
+    cmd.Flag.Bool("no-remove", false, "don't remove duplicate arch/name packages")
 
 	return cmd
 }

cmd/snapshot_pull.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"sort"
@@ -13,11 +14,12 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 4 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	noDeps := context.flags.Lookup("no-deps").Value.Get().(bool)
 	noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
+	allMatches := context.flags.Lookup("all-matches").Value.Get().(bool)
 
 	// Load <name> snapshot
 	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
@@ -75,77 +77,57 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
 	}
 
-	// Initial dependencies out of arguments
-	initialDependencies := make([]deb.Dependency, len(args)-3)
-	for i, arg := range args[3:] {
-		initialDependencies[i], err = deb.ParseDependency(arg)
-		if err != nil {
-			return fmt.Errorf("unable to parse argument: %s", err)
-		}
+	// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
+	var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
+	for _, arch := range architecturesList {
+		archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
 	}
 
-	// Perform pull
-	for _, arch := range architecturesList {
-		dependencies := make([]deb.Dependency, len(initialDependencies), 128)
-		for i := range dependencies {
-			dependencies[i] = initialDependencies[i]
-			dependencies[i].Architecture = arch
+	// Initial queries out of arguments
+	queries := make([]deb.PackageQuery, len(args)-3)
+	for i, arg := range args[3:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+		// Add architecture filter
+		queries[i] = &deb.AndQuery{queries[i], archQuery}
+	}
+
+	// Filter with dependencies as requested
+	result, err := sourcePackageList.Filter(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+	result.PrepareIndex()
+
+	alreadySeen := map[string]bool{}
+
+	result.ForEachIndexed(func(pkg *deb.Package) error {
+		key := pkg.Architecture + "_" + pkg.Name
+		_, seen := alreadySeen[key]
+
+		// If we haven't seen such name-architecture pair and were instructed to remove, remove it
+		if !noRemove && !seen {
+			// Remove all packages with the same name and architecture
+			pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
+			for _, p := range pS {
+				packageList.Remove(p)
+				context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
+			}
 		}
 
-		// Go over list of initial dependencies + list of dependencies found
-		for i := 0; i < len(dependencies); i++ {
-			dep := dependencies[i]
-
-			// Search for package that can satisfy dependencies
-			pkg := sourcePackageList.Search(dep)
-			if pkg == nil {
-				context.Progress().ColoredPrintf("@y[!]@| @!Dependency %s can't be satisfied with source %s@|", &dep, source)
-				continue
-			}
-
-			if !noRemove {
-				// Remove all packages with the same name and architecture
-				for p := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}); p != nil; {
-					packageList.Remove(p)
-					context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
-					p = packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name})
-				}
-			}
-
-			// Add new discovered package
+		// If !allMatches, add only first matching name-arch package
+		if !seen || allMatches {
 			packageList.Add(pkg)
 			context.Progress().ColoredPrintf("@g[+]@| %s added", pkg)
-
-			if noDeps {
-				continue
-			}
-
-			// Find missing dependencies for single added package
-			pL := deb.NewPackageList()
-			pL.Add(pkg)
-
-			var missing []deb.Dependency
-			missing, err = pL.VerifyDependencies(context.DependencyOptions(), []string{arch}, packageList, nil)
-			if err != nil {
-				context.Progress().ColoredPrintf("@y[!]@| @!Error while verifying dependencies for pkg %s: %s@|", pkg, err)
-			}
-
-			// Append missing dependencies to the list of dependencies to satisfy
-			for _, misDep := range missing {
-				found := false
-				for _, d := range dependencies {
-					if d == misDep {
-						found = true
-						break
-					}
-				}
-
-				if !found {
-					dependencies = append(dependencies, misDep)
-				}
-			}
 		}
-	}
+
+		alreadySeen[key] = true
+
+		return nil
+	})
+	alreadySeen = nil
 
 	if context.flags.Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nNot creating snapshot, as dry run was requested.\n")
@@ -167,14 +149,14 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 func makeCmdSnapshotPull() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotPull,
-		UsageLine: "pull <name> <source> <destination> <package-name> ...",
+		UsageLine: "pull <name> <source> <destination> <package-query> ...",
 		Short:     "pull packages from another snapshot",
 		Long: `
 Command pull pulls new packages along with its' dependencies to snapshot <name>
 from snapshot <source>. Pull can upgrade package version in <name> with
 versions from <source> following dependencies. New snapshot <destination>
 is created as a result of this process. Packages could be specified simply
-as 'package-name' or as dependency 'package-name (>= version)'.
+as 'package-name' or as package queries.
 
 Example:
 
@@ -186,6 +168,7 @@ Example:
 	cmd.Flag.Bool("dry-run", false, "don't create destination snapshot, just show what would be pulled")
 	cmd.Flag.Bool("no-deps", false, "don't process dependencies, just pull listed packages")
 	cmd.Flag.Bool("no-remove", false, "don't remove other package versions when pulling package")
+	cmd.Flag.Bool("all-matches", false, "pull all the packages that satisfy the dependency version requirements")
 
 	return cmd
 }

cmd/snapshot_rename.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlySnapshotRename(cmd *commander.Command, args []string) error {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().SnapshotCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: snapshot %s already exists", newName)
+	}
+
+	snapshot.Name = newName
+	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nSnapshot %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdSnapshotRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames snapshot",
+		Long: `
+Command changes name of the snapshot. Snapshot name should be unique.
+
+Example:
+
+  $ aptly snapshot rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/snapshot_show.go

@@ -10,7 +10,7 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]

cmd/snapshot_verify.go

@@ -11,7 +11,7 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	snapshots := make([]*deb.Snapshot, len(args))

cmd/version.go

@@ -7,6 +7,11 @@ import (
 )
 
 func aptlyVersion(cmd *commander.Command, args []string) error {
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
 	fmt.Printf("aptly version: %s\n", aptly.Version)
 	return nil
 }

deb/format.go

@@ -11,8 +11,9 @@ import (
 type Stanza map[string]string
 
 // Canonical order of fields in stanza
-var canocialOrder = []string{"Origin", "Label", "Suite", "Package", "Version", "Installed-Size", "Priority", "Section", "Maintainer",
-	"Architecture", "Codename", "Date", "Architectures", "Components", "Description", "MD5sum", "MD5Sum", "SHA1", "SHA256"}
+var canocialOrder = []string{"Package", "Origin", "Label", "Suite", "Version", "Installed-Size", "Priority", "Section", "Maintainer",
+	"Architecture", "Codename", "Date", "Architectures", "Components", "Description", "MD5sum", "MD5Sum", "SHA1", "SHA256",
+	"Archive", "Component"}
 
 // Copy returns copy of Stanza
 func (s Stanza) Copy() (result Stanza) {

deb/list.go

@@ -5,7 +5,6 @@ import (
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
 	"sort"
-	"strings"
 )
 
 // Dependency options
@@ -86,11 +85,11 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 
 // Add appends package to package list, additionally checking for uniqueness
 func (l *PackageList) Add(p *Package) error {
-	key := string(p.Key(""))
+	key := string(p.ShortKey(""))
 	existing, ok := l.packages[key]
 	if ok {
 		if !existing.Equals(p) {
-			return fmt.Errorf("conflict in package %s: %#v != %#v", p, existing, p)
+			return fmt.Errorf("conflict in package %s", p)
 		}
 		return nil
 	}
@@ -101,7 +100,7 @@ func (l *PackageList) Add(p *Package) error {
 			l.providesIndex[provides] = append(l.providesIndex[provides], p)
 		}
 
-		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= p.Name })
+		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.lessPackages(p, l.packagesIndex[j]) })
 
 		// insert p into l.packagesIndex in position i
 		l.packagesIndex = append(l.packagesIndex, nil)
@@ -123,6 +122,22 @@ func (l *PackageList) ForEach(handler func(*Package) error) error {
 	return err
 }
 
+// ForEachIndexed calls handler for each package in list in indexed order
+func (l *PackageList) ForEachIndexed(handler func(*Package) error) error {
+	if !l.indexed {
+		panic("list not indexed, can't iterate")
+	}
+
+	var err error
+	for _, p := range l.packagesIndex {
+		err = handler(p)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
 // Len returns number of packages in the list
 func (l *PackageList) Len() int {
 	return len(l.packages)
@@ -137,7 +152,7 @@ func (l *PackageList) Append(pl *PackageList) error {
 		existing, ok := l.packages[k]
 		if ok {
 			if !existing.Equals(p) {
-				return fmt.Errorf("conflict in package %s: %#v != %#v", p, existing, p)
+				return fmt.Errorf("conflict in package %s", p)
 			}
 		} else {
 			l.packages[k] = p
@@ -149,7 +164,7 @@ func (l *PackageList) Append(pl *PackageList) error {
 
 // Remove removes package from the list, and updates index when required
 func (l *PackageList) Remove(p *Package) {
-	delete(l.packages, string(p.Key("")))
+	delete(l.packages, string(p.ShortKey("")))
 	if l.indexed {
 		for _, provides := range p.Provides {
 			for i, pkg := range l.providesIndex[provides] {
@@ -263,7 +278,7 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 						continue
 					}
 
-					if sources.Search(dep) == nil {
+					if sources.Search(dep, false) == nil {
 						variantsMissing = append(variantsMissing, dep)
 						missingCount++
 					} else {
@@ -300,9 +315,21 @@ func (l *PackageList) Swap(i, j int) {
 	l.packagesIndex[i], l.packagesIndex[j] = l.packagesIndex[j], l.packagesIndex[i]
 }
 
-// Compare compares two names in lexographical order
+func (l *PackageList) lessPackages(iPkg, jPkg *Package) bool {
+	if iPkg.Name == jPkg.Name {
+		cmp := CompareVersions(iPkg.Version, jPkg.Version)
+		if cmp == 0 {
+			return iPkg.Architecture < jPkg.Architecture
+		}
+		return cmp == 1
+	}
+
+	return iPkg.Name < jPkg.Name
+}
+
+// Less compares two packages by name (lexographical) and version (latest to oldest)
 func (l *PackageList) Less(i, j int) bool {
-	return l.packagesIndex[i].Name < l.packagesIndex[j].Name
+	return l.lessPackages(l.packagesIndex[i], l.packagesIndex[j])
 }
 
 // PrepareIndex prepares list for indexing
@@ -325,16 +352,32 @@ func (l *PackageList) PrepareIndex() {
 	l.indexed = true
 }
 
-// Search searches package index for specified package
-func (l *PackageList) Search(dep Dependency) *Package {
+// Scan searches package index using full scan
+func (l *PackageList) Scan(q PackageQuery) (result *PackageList) {
+	result = NewPackageList()
+	for _, pkg := range l.packages {
+		if q.Matches(pkg) {
+			result.Add(pkg)
+		}
+	}
+
+	return
+}
+
+// Search searches package index for specified package(s) using optimized queries
+func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*Package) {
 	if !l.indexed {
 		panic("list not indexed, can't search")
 	}
 
 	if dep.Relation == VersionDontCare {
 		for _, p := range l.providesIndex[dep.Pkg] {
-			if p.MatchesArchitecture(dep.Architecture) {
-				return p
+			if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
+				searchResults = append(searchResults, p)
+
+				if !allMatches {
+					break
+				}
 			}
 		}
 	}
@@ -344,16 +387,21 @@ func (l *PackageList) Search(dep Dependency) *Package {
 	for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
 		p := l.packagesIndex[i]
 		if p.MatchesDependency(dep) {
-			return p
+			searchResults = append(searchResults, p)
+
+			if !allMatches {
+				break
+			}
 		}
 
 		i++
 	}
-	return nil
+
+	return
 }
 
 // Filter filters package index by specified queries (ORed together), possibly pulling dependencies
-func (l *PackageList) Filter(queries []string, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
+func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
 	if !l.indexed {
 		panic("list not indexed, can't filter")
 	}
@@ -361,53 +409,16 @@ func (l *PackageList) Filter(queries []string, withDependencies bool, source *Pa
 	result := NewPackageList()
 
 	for _, query := range queries {
-		isDepQuery := strings.IndexAny(query, " (){}=<>") != -1
-
-		if !isDepQuery {
-			// try to interpret query as package string representation
-
-			// convert Package.String() to Package.Key()
-			i := strings.Index(query, "_")
-			if i != -1 {
-				pkg, query := query[:i], query[i+1:]
-				j := strings.LastIndex(query, "_")
-				if j != -1 {
-					version, arch := query[:j], query[j+1:]
-					p := l.packages["P"+arch+" "+pkg+" "+version]
-					if p != nil {
-						result.Add(p)
-						continue
-					}
-				}
-			}
-		}
-
-		// try as dependency
-		dep, err := ParseDependency(query)
-		if err != nil {
-			if isDepQuery {
-				return nil, err
-			}
-			// parsing failed, but probably that wasn't a dep query
-			continue
-		}
-
-		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= dep.Pkg })
-
-		for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
-			p := l.packagesIndex[i]
-			if p.MatchesDependency(dep) {
-				result.Add(p)
-			}
-			i++
-		}
+		result.Append(query.Query(l))
 	}
 
 	if withDependencies {
 		added := result.Len()
 
 		dependencySource := NewPackageList()
-		dependencySource.Append(source)
+		if source != nil {
+			dependencySource.Append(source)
+		}
 		dependencySource.Append(result)
 		dependencySource.PrepareIndex()
 
@@ -423,11 +434,13 @@ func (l *PackageList) Filter(queries []string, withDependencies bool, source *Pa
 
 			// try to satisfy dependencies
 			for _, dep := range missing {
-				p := l.Search(dep)
-				if p != nil {
-					result.Add(p)
-					dependencySource.Add(p)
-					added++
+				searchResults := l.Search(dep, false)
+				if searchResults != nil {
+					for _, p := range searchResults {
+						result.Add(p)
+						dependencySource.Add(p)
+						added++
+					}
 				}
 			}
 		}

deb/list_test.go

@@ -3,10 +3,47 @@ package deb
 import (
 	"errors"
 	. "launchpad.net/gocheck"
+	"regexp"
 	"sort"
 	"strings"
 )
 
+type containsChecker struct {
+	*CheckerInfo
+}
+
+func (c *containsChecker) Check(params []interface{}, names []string) (result bool, error string) {
+	var (
+		pkgSlice1 []*Package
+		pkgSlice2 []*Package
+		ok        bool
+	)
+
+	pkgMap := make(map[*Package]bool)
+
+	pkgSlice1, ok = params[0].([]*Package)
+	if !ok {
+		return false, "The first parameter is not a Package slice"
+	}
+	pkgSlice2, ok = params[1].([]*Package)
+	if !ok {
+		return false, "The second parameter is not a Package slice"
+	}
+
+	for _, pkg := range pkgSlice2 {
+		pkgMap[pkg] = true
+	}
+
+	for _, pkg := range pkgSlice1 {
+		if _, ok := pkgMap[pkg]; !ok {
+			return false, ""
+		}
+	}
+	return true, ""
+}
+
+var Contains = &containsChecker{&CheckerInfo{Name: "Contains", Params: []string{"Container", "Expected to contain"}}}
+
 type PackageListSuite struct {
 	// Simple list with "real" packages from stanzas
 	list                   *PackageList
@@ -14,8 +51,10 @@ type PackageListSuite struct {
 
 	// Mocked packages in list
 	packages       []*Package
+	packages2      []*Package
 	sourcePackages []*Package
 	il             *PackageList
+	il2            *PackageList
 }
 
 var _ = Suite(&PackageListSuite{})
@@ -29,7 +68,7 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 	stanza["Package"] = "mars-invaders"
 	s.p3 = NewPackageFromControlFile(stanza)
 	stanza = packageStanza.Copy()
-	stanza["Size"] = "42"
+	stanza["Source"] = "unknown-planet"
 	s.p4 = NewPackageFromControlFile(stanza)
 	stanza = packageStanza.Copy()
 	stanza["Package"] = "lonely-strangers"
@@ -60,6 +99,20 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 	}
 	s.il.PrepareIndex()
 
+	s.il2 = NewPackageList()
+	s.packages2 = []*Package{
+		&Package{Name: "mailer", Version: "3.5.8", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		&Package{Name: "sendmail", Version: "1.0", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		&Package{Name: "app", Version: "1.1-bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		&Package{Name: "app", Version: "1.1-bp2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		&Package{Name: "app", Version: "1.2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
+		&Package{Name: "app", Version: "3.0", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+	}
+	for _, p := range s.packages2 {
+		s.il2.Add(p)
+	}
+	s.il2.PrepareIndex()
+
 	s.sourcePackages = []*Package{
 		&Package{Name: "postfix", Version: "1.3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
 		&Package{Name: "app", Version: "1.1~bp1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
@@ -196,35 +249,60 @@ func (s *PackageListSuite) TestAppend(c *C) {
 }
 
 func (s *PackageListSuite) TestSearch(c *C) {
-	c.Check(func() { s.list.Search(Dependency{Architecture: "i386", Pkg: "app"}) }, Panics, "list not indexed, can't search")
+	//allMatches = False
+	c.Check(func() { s.list.Search(Dependency{Architecture: "i386", Pkg: "app"}, false) }, Panics, "list not indexed, can't search")
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "mail-agent"}), Equals, s.packages[4])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "puppy"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "mail-agent"}, false), DeepEquals, []*Package{s.packages[4]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "puppy"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp2"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp2"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1~~"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1~~"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~bp1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~~"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~~"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.0"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.2"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.0"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.2"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}, false), IsNil)
+
+	// search w/o version should return package with latest version
+	c.Check(s.il.Search(Dependency{Architecture: "source", Pkg: "dpkg"}, false), DeepEquals, []*Package{s.packages[13]})
+
+	// allMatches = True
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "mail-agent"}, true), Contains, []*Package{s.packages2[0], s.packages2[1]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "puppy"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "3"}, true), Contains, []*Package{s.packages2[5]})
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLess, Version: "1.2"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLess, Version: "1.1~"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.2"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1-bp1"}, true), Contains, []*Package{s.packages2[2]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.0"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreater, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreater, Version: "5.0"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}, true), Contains, []*Package{s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "5.0"}, true), IsNil)
 }
 
 func (s *PackageListSuite) TestFilter(c *C) {
-	c.Check(func() { s.list.Filter([]string{"abcd_0.3_i386"}, false, nil, 0, nil) }, Panics, "list not indexed, can't filter")
-
-	_, err := s.il.Filter([]string{"app >3)"}, false, nil, 0, nil)
-	c.Check(err, ErrorMatches, "unable to parse dependency.*")
+	c.Check(func() { s.list.Filter([]PackageQuery{&PkgQuery{"abcd", "0.3", "i386"}}, false, nil, 0, nil) }, Panics, "list not indexed, can't filter")
 
 	plString := func(l *PackageList) string {
 		list := make([]string, 0, l.Len())
@@ -237,25 +315,69 @@ func (s *PackageListSuite) TestFilter(c *C) {
 		return strings.Join(list, " ")
 	}
 
-	result, err := s.il.Filter([]string{"app_1.1~bp1_i386"}, false, nil, 0, nil)
+	result, err := s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
 
-	result, err = s.il.Filter([]string{"app_1.1~bp1_i386", "dpkg_1.7_source", "dpkg_1.8_amd64"}, false, nil, 0, nil)
+	result, err = s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, &PkgQuery{"dpkg", "1.7", "source"},
+		&PkgQuery{"dpkg", "1.8", "amd64"}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_source")
 
-	result, err = s.il.Filter([]string{"app", "dpkg (>>1.6.1-3)", "app (>=1.0)", "xyz", "aa (>>3.0)"}, false, nil, 0, nil)
+	result, err = s.il.Filter([]PackageQuery{
+		&DependencyQuery{Dep: Dependency{Pkg: "app"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "xyz"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "aa", Relation: VersionGreater, Version: "3.0"}}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source")
 
-	result, err = s.il.Filter([]string{"app {i386}"}, true, NewPackageList(), 0, []string{"i386"})
+	result, err = s.il.Filter([]PackageQuery{&DependencyQuery{Dep: Dependency{Pkg: "app", Architecture: "i386"}}}, true, NewPackageList(), 0, []string{"i386"})
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386")
 
-	result, err = s.il.Filter([]string{"app (>=0.9)", "lib", "data"}, true, NewPackageList(), 0, []string{"i386", "amd64"})
+	result, err = s.il.Filter([]PackageQuery{
+		&DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "0.9"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "lib"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "data"}}}, true, NewPackageList(), 0, []string{"i386", "amd64"})
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "")
+
+	result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_i386 data_1.1~bp1_all")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{&FieldQuery{Field: "Version", Relation: VersionGreaterOrEqual, Value: "1.0"},
+		&FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 data_1.1~bp1_all")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&NotQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.6.1-3_arm dpkg_1.6.1-3_source dpkg_1.7_source libx_1.5_arm")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionRegexp, Value: "i.*6", Regexp: regexp.MustCompile("i.*6")}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
 }
 
 func (s *PackageListSuite) TestVerifyDependencies(c *C) {

deb/package.go

@@ -26,6 +26,8 @@ type Package struct {
 	IsSource bool
 	// Hash of files section
 	FilesHash uint64
+	// Is this >= 0.6 package?
+	V06Plus bool
 	// Offload fields
 	deps  *PackageDependencies
 	extra *Stanza
@@ -41,6 +43,7 @@ func NewPackageFromControlFile(input Stanza) *Package {
 		Version:      input["Version"],
 		Architecture: input["Architecture"],
 		Source:       input["Source"],
+		V06Plus:      true,
 	}
 
 	delete(input, "Package")
@@ -89,6 +92,7 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 		Version:            input["Version"],
 		Architecture:       "source",
 		SourceArchitecture: input["Architecture"],
+		V06Plus:            true,
 	}
 
 	delete(input, "Package")
@@ -167,7 +171,16 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 
 // Key returns unique key identifying package
 func (p *Package) Key(prefix string) []byte {
-	return []byte(prefix + "P" + p.Architecture + " " + p.Name + " " + p.Version)
+	if p.V06Plus {
+		return []byte(fmt.Sprintf("%sP%s %s %s %08x", prefix, p.Architecture, p.Name, p.Version, p.FilesHash))
+	}
+
+	return p.ShortKey(prefix)
+}
+
+// ShortKey returns key for the package that should be unique in one list
+func (p *Package) ShortKey(prefix string) []byte {
+	return []byte(fmt.Sprintf("%sP%s %s %s", prefix, p.Architecture, p.Name, p.Version))
 }
 
 // String creates readable representation
@@ -175,6 +188,70 @@ func (p *Package) String() string {
 	return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.Architecture)
 }
 
+// GetField returns fields from package
+func (p *Package) GetField(name string) string {
+	switch name {
+	// $Version is handled in FieldQuery
+	case "$Source":
+		if p.IsSource {
+			return ""
+		}
+		source := p.Source
+		if source == "" {
+			return p.Name
+		} else if pos := strings.Index(source, "("); pos != -1 {
+			return strings.TrimSpace(source[:pos])
+		}
+		return source
+	case "$SourceVersion":
+		if p.IsSource {
+			return ""
+		}
+		source := p.Source
+		if pos := strings.Index(source, "("); pos != -1 {
+			if pos2 := strings.LastIndex(source, ")"); pos2 != -1 && pos2 > pos {
+				return strings.TrimSpace(source[pos+1 : pos2])
+			}
+		}
+		return p.Version
+	case "$Architecture":
+		return p.Architecture
+	case "$PackageType":
+		if p.IsSource {
+			return "source"
+		}
+		return "deb"
+	case "Name":
+		return p.Name
+	case "Version":
+		return p.Version
+	case "Architecture":
+		if p.IsSource {
+			return p.SourceArchitecture
+		}
+		return p.Architecture
+	case "Source":
+		return p.Source
+	case "Depends":
+		return strings.Join(p.Deps().Depends, ", ")
+	case "Pre-Depends":
+		return strings.Join(p.Deps().PreDepends, ", ")
+	case "Suggests":
+		return strings.Join(p.Deps().Suggests, ", ")
+	case "Recommends":
+		return strings.Join(p.Deps().Recommends, ", ")
+	case "Provides":
+		return strings.Join(p.Provides, ", ")
+	case "Build-Depends":
+		return strings.Join(p.Deps().BuildDepends, ", ")
+	case "Build-Depends-Indep":
+		return strings.Join(p.Deps().BuildDependsInDep, ", ")
+	default:
+		return p.Extra()[name]
+	}
+	return ""
+}
+
 // MatchesArchitecture checks whether packages matches specified architecture
 func (p *Package) MatchesArchitecture(arch string) bool {
 	if p.Architecture == "all" && arch != "source" {
@@ -186,19 +263,23 @@ func (p *Package) MatchesArchitecture(arch string) bool {
 
 // MatchesDependency checks whether package matches specified dependency
 func (p *Package) MatchesDependency(dep Dependency) bool {
-	if dep.Pkg != p.Name {
-		return false
-	}
-
 	if dep.Architecture != "" && !p.MatchesArchitecture(dep.Architecture) {
 		return false
 	}
 
 	if dep.Relation == VersionDontCare {
-		return true
+		if utils.StrSliceHasItem(p.Provides, dep.Pkg) {
+			return true
+		}
+		return dep.Pkg == p.Name
+	}
+
+	if dep.Pkg != p.Name {
+		return false
 	}
 
 	r := CompareVersions(p.Version, dep.Version)
+
 	switch dep.Relation {
 	case VersionEqual:
 		return r == 0
@@ -210,6 +291,11 @@ func (p *Package) MatchesDependency(dep Dependency) bool {
 		return r <= 0
 	case VersionGreaterOrEqual:
 		return r >= 0
+	case VersionPatternMatch:
+		matched, err := filepath.Match(dep.Version, p.Version)
+		return err == nil && matched
+	case VersionRegexp:
+		return dep.Regexp.FindStringIndex(p.Version) != nil
 	}
 
 	panic("unknown relation")
@@ -376,7 +462,8 @@ func (p *Package) Equals(p2 *Package) bool {
 }
 
 // LinkFromPool links package file from pool to dist's pool location
-func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool, prefix string, component string) error {
+func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool,
+	prefix, component string, force bool) error {
 	poolDir, err := p.PoolDirectory()
 	if err != nil {
 		return err
@@ -391,7 +478,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 		relPath := filepath.Join("pool", component, poolDir)
 		publishedDirectory := filepath.Join(prefix, relPath)
 
-		err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePath)
+		err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePath, f.Checksums.MD5, force)
 		if err != nil {
 			return err
 		}

deb/package_collection.go

@@ -12,12 +12,14 @@ import (
 type PackageCollection struct {
 	db           database.Storage
 	encodeBuffer bytes.Buffer
+	codecHandle  *codec.MsgpackHandle
 }
 
 // NewPackageCollection creates new PackageCollection and binds it to database
 func NewPackageCollection(db database.Storage) *PackageCollection {
 	return &PackageCollection{
-		db: db,
+		db:          db,
+		codecHandle: &codec.MsgpackHandle{},
 	}
 }
 
@@ -53,7 +55,7 @@ func (collection *PackageCollection) ByKey(key []byte) (*Package, error) {
 	if len(encoded) > 2 && (encoded[0] != 0xc1 || encoded[1] != 0x1) {
 		oldp := &oldPackage{}
 
-		decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+		decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 		err = decoder.Decode(oldp)
 		if err != nil {
 			return nil, err
@@ -83,12 +85,12 @@ func (collection *PackageCollection) ByKey(key []byte) (*Package, error) {
 		p.UpdateFiles(PackageFiles(oldp.Files))
 
 		// Save in new format
-		err = collection.internalUpdate(p)
+		err = collection.Update(p)
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		decoder := codec.NewDecoderBytes(encoded[2:], &codec.MsgpackHandle{})
+		decoder := codec.NewDecoderBytes(encoded[2:], collection.codecHandle)
 		err = decoder.Decode(p)
 		if err != nil {
 			return nil, err
@@ -109,7 +111,7 @@ func (collection *PackageCollection) loadExtra(p *Package) *Stanza {
 
 	stanza := &Stanza{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(stanza)
 	if err != nil {
 		panic("unable to decode extra")
@@ -127,7 +129,7 @@ func (collection *PackageCollection) loadDependencies(p *Package) *PackageDepend
 
 	deps := &PackageDependencies{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(deps)
 	if err != nil {
 		panic("unable to decode deps")
@@ -145,7 +147,7 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 
 	files := &PackageFiles{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(files)
 	if err != nil {
 		panic("unable to decode files")
@@ -156,26 +158,7 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 
 // Update adds or updates information about package in DB checking for conficts first
 func (collection *PackageCollection) Update(p *Package) error {
-	existing, err := collection.ByKey(p.Key(""))
-	if err == nil {
-		// if .Files is different, consider to be conflict
-		if p.FilesHash != existing.FilesHash {
-			return fmt.Errorf("unable to save: %s, conflict with existing packge", p)
-		}
-		// ok, .Files are the same, but maybe some meta-data is different, proceed to saving
-	} else {
-		if err != database.ErrNotFound {
-			return err
-		}
-		// ok, package doesn't exist yet
-	}
-
-	return collection.internalUpdate(p)
-}
-
-// internalUpdate updates information in DB about package and offloaded fields
-func (collection *PackageCollection) internalUpdate(p *Package) error {
-	encoder := codec.NewEncoder(&collection.encodeBuffer, &codec.MsgpackHandle{})
+	encoder := codec.NewEncoder(&collection.encodeBuffer, collection.codecHandle)
 
 	collection.encodeBuffer.Reset()
 	collection.encodeBuffer.WriteByte(0xc1)

deb/package_collection_test.go

@@ -48,20 +48,6 @@ func (s *PackageCollectionSuite) TestUpdate(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(res.Equals(s.p), Equals, false)
 	c.Assert(res.Equals(p2), Equals, true)
-
-	// change file info
-	p2 = NewPackageFromControlFile(packageStanza.Copy())
-	p2.UpdateFiles(nil)
-	res, err = s.collection.ByKey(p2.Key(""))
-	err = s.collection.Update(p2)
-	c.Assert(err, ErrorMatches, ".*conflict with existing packge")
-	p2 = NewPackageFromControlFile(packageStanza.Copy())
-	files := p2.Files()
-	files[0].Checksums.MD5 = "abcdef"
-	p2.UpdateFiles(files)
-	res, err = s.collection.ByKey(p2.Key(""))
-	err = s.collection.Update(p2)
-	c.Assert(err, ErrorMatches, ".*conflict with existing packge")
 }
 
 func (s *PackageCollectionSuite) TestByKey(c *C) {

deb/package_test.go

@@ -7,6 +7,7 @@ import (
 	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
+	"regexp"
 )
 
 type PackageSuite struct {
@@ -87,10 +88,21 @@ func (s *PackageSuite) TestWithProvides(c *C) {
 func (s *PackageSuite) TestKey(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
+	c.Check(p.Key(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2 c8901eedd79ac51b"))
+	c.Check(p.Key("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2 c8901eedd79ac51b"))
+
+	p.V06Plus = false
 	c.Check(p.Key(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2"))
 	c.Check(p.Key("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2"))
 }
 
+func (s *PackageSuite) TestShortKey(c *C) {
+	p := NewPackageFromControlFile(s.stanza)
+
+	c.Check(p.ShortKey(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2"))
+	c.Check(p.ShortKey("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2"))
+}
+
 func (s *PackageSuite) TestStanza(c *C) {
 	p := NewPackageFromControlFile(s.stanza.Copy())
 	stanza := p.Stanza()
@@ -108,6 +120,58 @@ func (s *PackageSuite) TestString(c *C) {
 	c.Assert(p.String(), Equals, "alien-arena-common_7.40-2_i386")
 }
 
+func (s *PackageSuite) TestGetField(c *C) {
+	p := NewPackageFromControlFile(s.stanza.Copy())
+
+	stanza2 := s.stanza.Copy()
+	delete(stanza2, "Source")
+	stanza2["Provides"] = "app, game"
+	p2 := NewPackageFromControlFile(stanza2)
+
+	stanza3 := s.stanza.Copy()
+	stanza3["Source"] = "alien-arena (3.5)"
+	p3 := NewPackageFromControlFile(stanza3)
+
+	p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
+
+	c.Check(p.GetField("$Source"), Equals, "alien-arena")
+	c.Check(p2.GetField("$Source"), Equals, "alien-arena-common")
+	c.Check(p3.GetField("$Source"), Equals, "alien-arena")
+	c.Check(p4.GetField("$Source"), Equals, "")
+
+	c.Check(p.GetField("$SourceVersion"), Equals, "7.40-2")
+	c.Check(p2.GetField("$SourceVersion"), Equals, "7.40-2")
+	c.Check(p3.GetField("$SourceVersion"), Equals, "3.5")
+	c.Check(p4.GetField("$SourceVersion"), Equals, "")
+
+	c.Check(p.GetField("$Architecture"), Equals, "i386")
+	c.Check(p4.GetField("$Architecture"), Equals, "source")
+
+	c.Check(p.GetField("$PackageType"), Equals, "deb")
+	c.Check(p4.GetField("$PackageType"), Equals, "source")
+
+	c.Check(p.GetField("Name"), Equals, "alien-arena-common")
+	c.Check(p4.GetField("Name"), Equals, "access-modifier-checker")
+
+	c.Check(p.GetField("Architecture"), Equals, "i386")
+	c.Check(p4.GetField("Architecture"), Equals, "all")
+
+	c.Check(p.GetField("Version"), Equals, "7.40-2")
+
+	c.Check(p.GetField("Source"), Equals, "alien-arena")
+	c.Check(p2.GetField("Source"), Equals, "")
+	c.Check(p3.GetField("Source"), Equals, "alien-arena (3.5)")
+	c.Check(p4.GetField("Source"), Equals, "")
+
+	c.Check(p.GetField("Depends"), Equals, "libc6 (>= 2.7), alien-arena-data (>= 7.40)")
+
+	c.Check(p.GetField("Provides"), Equals, "")
+	c.Check(p2.GetField("Provides"), Equals, "app, game")
+
+	c.Check(p.GetField("Section"), Equals, "contrib/games")
+	c.Check(p.GetField("Priority"), Equals, "extra")
+}
+
 func (s *PackageSuite) TestEquals(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
@@ -163,6 +227,9 @@ func (s *PackageSuite) TestMatchesDependency(c *C) {
 	// exact match
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionEqual, Version: "7.40-2"}), Equals, true)
 
+	// exact match, same version, no revision specified
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionEqual, Version: "7.40"}), Equals, false)
+
 	// different name
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena", Architecture: "i386", Relation: VersionEqual, Version: "7.40-2"}), Equals, false)
 
@@ -193,6 +260,29 @@ func (s *PackageSuite) TestMatchesDependency(c *C) {
 	// <=
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionLessOrEqual, Version: "7.40-2"}), Equals, true)
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionLessOrEqual, Version: "7.40-1"}), Equals, false)
+
+	// %
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-*"}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-[2]"}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-[2"}), Equals, false)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-[34]"}), Equals, false)
+
+	// ~
+	c.Check(
+		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
+			Regexp: regexp.MustCompile("7\\.40-.*")}), Equals, true)
+	c.Check(
+		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
+			Regexp: regexp.MustCompile("40")}), Equals, true)
+	c.Check(
+		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
+			Regexp: regexp.MustCompile("39-.*")}), Equals, false)
+
+	// Provides
+	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Relation: VersionDontCare}), Equals, false)
+	p.Provides = []string{"fun", "game"}
+	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Relation: VersionDontCare}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Architecture: "amd64", Relation: VersionDontCare}), Equals, false)
 }
 
 func (s *PackageSuite) TestGetDependencies(c *C) {
@@ -255,13 +345,13 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
 	c.Assert(err, IsNil)
 	file.Close()
 
-	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free")
+	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
 	c.Check(err, IsNil)
 	c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
 	c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena")
 
 	p.IsSource = true
-	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free")
+	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
 	c.Check(err, IsNil)
 	c.Check(p.Extra()["Directory"], Equals, "pool/non-free/a/alien-arena")
 }

deb/publish_test.go

@@ -1,10 +1,14 @@
 package deb
 
 import (
+	"bytes"
 	"errors"
+	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/files"
+	"github.com/ugorji/go/codec"
+	"io/ioutil"
 	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
@@ -36,24 +40,37 @@ func (n *NullSigner) SetKeyRing(keyring, secretKeyring string) {
 }
 
 func (n *NullSigner) DetachedSign(source string, destination string) error {
-	return nil
+	return ioutil.WriteFile(destination, []byte{}, 0644)
 }
 
 func (n *NullSigner) ClearSign(source string, destination string) error {
-	return nil
+	return ioutil.WriteFile(destination, []byte{}, 0644)
+}
+
+type FakeStorageProvider struct {
+	storages map[string]aptly.PublishedStorage
+}
+
+func (p *FakeStorageProvider) GetPublishedStorage(name string) aptly.PublishedStorage {
+	storage, ok := p.storages[name]
+	if !ok {
+		panic(fmt.Sprintf("unknown storage: %#v", name))
+	}
+	return storage
 }
 
 type PublishedRepoSuite struct {
 	PackageListMixinSuite
-	repo, repo2       *PublishedRepo
-	root              string
-	publishedStorage  aptly.PublishedStorage
-	packagePool       aptly.PackagePool
-	localRepo         *LocalRepo
-	snapshot          *Snapshot
-	db                database.Storage
-	factory           *CollectionFactory
-	packageCollection *PackageCollection
+	repo, repo2, repo3, repo4, repo5    *PublishedRepo
+	root, root2                         string
+	provider                            *FakeStorageProvider
+	publishedStorage, publishedStorage2 *files.PublishedStorage
+	packagePool                         aptly.PackagePool
+	localRepo                           *LocalRepo
+	snapshot, snapshot2                 *Snapshot
+	db                                  database.Storage
+	factory                             *CollectionFactory
+	packageCollection                   *PackageCollection
 }
 
 var _ = Suite(&PublishedRepoSuite{})
@@ -66,6 +83,11 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.root = c.MkDir()
 	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.root2 = c.MkDir()
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{
+		"":            s.publishedStorage,
+		"files:other": s.publishedStorage2}}
 	s.packagePool = files.NewPackagePool(s.root)
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
@@ -79,14 +101,23 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.snapshot, _ = NewSnapshotFromRepository("snap", repo)
 	s.factory.SnapshotCollection().Add(s.snapshot)
 
+	s.snapshot2, _ = NewSnapshotFromRepository("snap", repo)
+	s.factory.SnapshotCollection().Add(s.snapshot2)
+
 	s.packageCollection = s.factory.PackageCollection()
 	s.packageCollection.Update(s.p1)
 	s.packageCollection.Update(s.p2)
 	s.packageCollection.Update(s.p3)
 
-	s.repo, _ = NewPublishedRepo("ppa", "squeeze", "main", nil, s.snapshot, s.factory)
+	s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
 
-	s.repo2, _ = NewPublishedRepo("ppa", "maverick", "main", nil, s.localRepo, s.factory)
+	s.repo2, _ = NewPublishedRepo("", "ppa", "maverick", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+
+	s.repo3, _ = NewPublishedRepo("", "linux", "natty", nil, []string{"main", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+
+	s.repo4, _ = NewPublishedRepo("", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 
 	poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
 	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
@@ -100,17 +131,39 @@ func (s *PublishedRepoSuite) TearDownTest(c *C) {
 }
 
 func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) {
-	c.Check(s.repo.snapshot, Equals, s.snapshot)
+	c.Check(s.repo.sourceItems["main"].snapshot, Equals, s.snapshot)
 	c.Check(s.repo.SourceKind, Equals, "snapshot")
-	c.Check(s.repo.SourceUUID, Equals, s.snapshot.UUID)
+	c.Check(s.repo.Sources["main"], Equals, s.snapshot.UUID)
+	c.Check(s.repo.Components(), DeepEquals, []string{"main"})
 
-	c.Check(s.repo2.localRepo, Equals, s.localRepo)
+	c.Check(s.repo2.sourceItems["main"].localRepo, Equals, s.localRepo)
 	c.Check(s.repo2.SourceKind, Equals, "local")
-	c.Check(s.repo2.SourceUUID, Equals, s.localRepo.UUID)
-	c.Check(s.repo2.packageRefs.Len(), Equals, 3)
+	c.Check(s.repo2.Sources["main"], Equals, s.localRepo.UUID)
+	c.Check(s.repo2.sourceItems["main"].packageRefs.Len(), Equals, 3)
+	c.Check(s.repo2.Components(), DeepEquals, []string{"main"})
 
-	c.Check(s.repo.RefList().Len(), Equals, 3)
-	c.Check(s.repo2.RefList().Len(), Equals, 3)
+	c.Check(s.repo.RefList("main").Len(), Equals, 3)
+	c.Check(s.repo2.RefList("main").Len(), Equals, 3)
+
+	c.Check(s.repo3.Sources, DeepEquals, map[string]string{"main": s.snapshot.UUID, "contrib": s.snapshot2.UUID})
+	c.Check(s.repo3.SourceKind, Equals, "snapshot")
+	c.Check(s.repo3.sourceItems["main"].snapshot, Equals, s.snapshot)
+	c.Check(s.repo3.sourceItems["contrib"].snapshot, Equals, s.snapshot2)
+	c.Check(s.repo3.Components(), DeepEquals, []string{"contrib", "main"})
+
+	c.Check(s.repo3.RefList("main").Len(), Equals, 3)
+	c.Check(s.repo3.RefList("contrib").Len(), Equals, 3)
+
+	c.Check(func() { NewPublishedRepo("", ".", "a", nil, nil, nil, s.factory) }, PanicMatches, "publish with empty sources")
+	c.Check(func() {
+		NewPublishedRepo("", ".", "a", nil, []string{"main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	}, PanicMatches, "sources and components should be equal in size")
+	c.Check(func() {
+		NewPublishedRepo("", ".", "a", nil, []string{"main", "contrib"}, []interface{}{s.localRepo, s.snapshot2}, s.factory)
+	}, PanicMatches, "interface conversion:.*")
+
+	_, err := NewPublishedRepo("", ".", "a", nil, []string{"main", "main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	c.Check(err, ErrorMatches, "duplicate component name: main")
 }
 
 func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
@@ -169,7 +222,7 @@ func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
 			errorExpected: "invalid prefix .*",
 		},
 	} {
-		repo, err := NewPublishedRepo(t.prefix, "squeeze", "main", nil, s.snapshot, s.factory)
+		repo, err := NewPublishedRepo("", t.prefix, "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
 		if t.errorExpected != "" {
 			c.Check(err, ErrorMatches, t.errorExpected)
 		} else {
@@ -179,41 +232,49 @@ func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
 }
 
 func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
-	repo, err := NewPublishedRepo("ppa", "", "", nil, s.snapshot, s.factory)
+	repo, err := NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "squeeze")
-	c.Check(repo.Component, Equals, "main")
+	c.Check(repo.Components(), DeepEquals, []string{"main"})
 
-	repo, err = NewPublishedRepo("ppa", "wheezy", "", nil, s.snapshot, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "wheezy", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "wheezy")
-	c.Check(repo.Component, Equals, "main")
+	c.Check(repo.Components(), DeepEquals, []string{"main"})
 
-	repo, err = NewPublishedRepo("ppa", "", "non-free", nil, s.snapshot, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"non-free"}, []interface{}{s.snapshot}, s.factory)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "squeeze")
-	c.Check(repo.Component, Equals, "non-free")
+	c.Check(repo.Components(), DeepEquals, []string{"non-free"})
 
-	repo, err = NewPublishedRepo("ppa", "squeeze", "", nil, s.localRepo, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "squeeze", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "squeeze")
-	c.Check(repo.Component, Equals, "main")
+	c.Check(repo.Components(), DeepEquals, []string{"main"})
 
-	repo, err = NewPublishedRepo("ppa", "", "main", nil, s.localRepo, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 	c.Check(err, ErrorMatches, "unable to guess distribution name, please specify explicitly")
 
 	s.localRepo.DefaultDistribution = "precise"
 	s.localRepo.DefaultComponent = "contrib"
 	s.factory.LocalRepoCollection().Update(s.localRepo)
 
-	repo, err = NewPublishedRepo("ppa", "", "", nil, s.localRepo, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "precise")
-	c.Check(repo.Component, Equals, "contrib")
+	c.Check(repo.Components(), DeepEquals, []string{"contrib"})
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "squeeze")
+	c.Check(repo.Components(), DeepEquals, []string{"contrib", "main"})
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	c.Check(err, ErrorMatches, "duplicate component name: main")
 }
 
 func (s *PublishedRepoSuite) TestPublish(c *C) {
-	err := s.repo.Publish(s.packagePool, s.publishedStorage, s.factory, &NullSigner{}, nil)
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false)
 	c.Assert(err, IsNil)
 
 	c.Check(s.repo.Architectures, DeepEquals, []string{"i386"})
@@ -245,49 +306,83 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(st, IsNil)
 
+	drf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"))
+	c.Assert(err, IsNil)
+
+	cfr = NewControlFileReader(drf)
+	st, err = cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(st["Archive"], Equals, "squeeze")
+	c.Check(st["Architecture"], Equals, "i386")
+
 	_, err = os.Stat(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb"))
 	c.Assert(err, IsNil)
 }
 
 func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
-	err := s.repo.Publish(s.packagePool, s.publishedStorage, s.factory, nil, nil)
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
 	c.Assert(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"), PathExists)
 }
 
 func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) {
-	err := s.repo2.Publish(s.packagePool, s.publishedStorage, s.factory, nil, nil)
+	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
 	c.Assert(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/main/binary-i386/Release"), PathExists)
+}
+
+func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) {
+	err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	c.Assert(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/main/source/Release"), PathExists)
+}
+
+func (s *PublishedRepoSuite) TestPublishOtherStorage(c *C) {
+	err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	c.Assert(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), Not(PathExists))
 }
 
 func (s *PublishedRepoSuite) TestString(c *C) {
 	c.Check(s.repo.String(), Equals,
-		"ppa/squeeze (main) [] publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
+		"ppa/squeeze [] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
 	c.Check(s.repo2.String(), Equals,
-		"ppa/maverick (main) [] publishes [local1]: comment1")
-	repo, _ := NewPublishedRepo("", "squeeze", "main", []string{"s390"}, s.snapshot, s.factory)
+		"ppa/maverick [] publishes {main: [local1]: comment1}")
+	repo, _ := NewPublishedRepo("", "", "squeeze", []string{"s390"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
 	c.Check(repo.String(), Equals,
-		"./squeeze (main) [s390] publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
-	repo, _ = NewPublishedRepo("", "squeeze", "main", []string{"i386", "amd64"}, s.snapshot, s.factory)
+		"./squeeze [s390] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	repo, _ = NewPublishedRepo("", "", "squeeze", []string{"i386", "amd64"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
 	c.Check(repo.String(), Equals,
-		"./squeeze (main) [i386, amd64] publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
+		"./squeeze [i386, amd64] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
 	repo.Origin = "myorigin"
 	c.Check(repo.String(), Equals,
-		"./squeeze (main, origin: myorigin) [i386, amd64] publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
+		"./squeeze (origin: myorigin) [i386, amd64] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
 	repo.Label = "mylabel"
 	c.Check(repo.String(), Equals,
-		"./squeeze (main, origin: myorigin, label: mylabel) [i386, amd64] publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
+		"./squeeze (origin: myorigin, label: mylabel) [i386, amd64] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	c.Check(s.repo3.String(), Equals,
+		"linux/natty [] publishes {contrib: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}, {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	c.Check(s.repo5.String(), Equals,
+		"files:other:ppa/maverick [source] publishes {main: [local1]: comment1}")
 }
 
 func (s *PublishedRepoSuite) TestKey(c *C) {
 	c.Check(s.repo.Key(), DeepEquals, []byte("Uppa>>squeeze"))
+	c.Check(s.repo5.Key(), DeepEquals, []byte("Ufiles:other:ppa>>maverick"))
 }
 
 func (s *PublishedRepoSuite) TestRefKey(c *C) {
-	c.Check(s.repo.RefKey(), DeepEquals, []byte("E"+s.repo.UUID))
+	c.Check(s.repo.RefKey(""), DeepEquals, []byte("E"+s.repo.UUID))
+	c.Check(s.repo.RefKey("main"), DeepEquals, []byte("E"+s.repo.UUID+"main"))
 }
 
 func (s *PublishedRepoSuite) TestEncodeDecode(c *C) {
@@ -295,7 +390,7 @@ func (s *PublishedRepoSuite) TestEncodeDecode(c *C) {
 	repo := &PublishedRepo{}
 	err := repo.Decode(encoded)
 
-	s.repo.snapshot = nil
+	s.repo.sourceItems = nil
 	c.Assert(err, IsNil)
 	c.Assert(repo, DeepEquals, s.repo)
 
@@ -303,21 +398,20 @@ func (s *PublishedRepoSuite) TestEncodeDecode(c *C) {
 	repo2 := &PublishedRepo{}
 	err = repo2.Decode(encoded2)
 
-	s.repo2.localRepo = nil
-	s.repo2.packageRefs = nil
+	s.repo2.sourceItems = nil
 	c.Assert(err, IsNil)
 	c.Assert(repo2, DeepEquals, s.repo2)
 }
 
 type PublishedRepoCollectionSuite struct {
 	PackageListMixinSuite
-	db                         database.Storage
-	factory                    *CollectionFactory
-	snapshotCollection         *SnapshotCollection
-	collection                 *PublishedRepoCollection
-	snap1, snap2               *Snapshot
-	localRepo                  *LocalRepo
-	repo1, repo2, repo3, repo4 *PublishedRepo
+	db                                database.Storage
+	factory                           *CollectionFactory
+	snapshotCollection                *SnapshotCollection
+	collection                        *PublishedRepoCollection
+	snap1, snap2                      *Snapshot
+	localRepo                         *LocalRepo
+	repo1, repo2, repo3, repo4, repo5 *PublishedRepo
 }
 
 var _ = Suite(&PublishedRepoCollectionSuite{})
@@ -337,10 +431,11 @@ func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
 	s.localRepo = NewLocalRepo("local1", "comment1")
 	s.factory.LocalRepoCollection().Add(s.localRepo)
 
-	s.repo1, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap1, s.factory)
-	s.repo2, _ = NewPublishedRepo("", "anaconda", "main", []string{}, s.snap2, s.factory)
-	s.repo3, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap2, s.factory)
-	s.repo4, _ = NewPublishedRepo("ppa", "precise", "main", []string{}, s.localRepo, s.factory)
+	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main", "contrib"}, []interface{}{s.snap2, s.snap1}, s.factory)
+	s.repo3, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap2}, s.factory)
+	s.repo4, _ = NewPublishedRepo("", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 
 	s.collection = s.factory.PublishedRepoCollection()
 }
@@ -349,8 +444,8 @@ func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) {
 	s.db.Close()
 }
 
-func (s *PublishedRepoCollectionSuite) TestAddByPrefixDistribution(c *C) {
-	r, err := s.collection.ByPrefixDistribution("ppa", "anaconda")
+func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C) {
+	r, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	c.Assert(s.collection.Add(s.repo1), IsNil)
@@ -360,8 +455,9 @@ func (s *PublishedRepoCollectionSuite) TestAddByPrefixDistribution(c *C) {
 	c.Assert(s.collection.Add(s.repo3), ErrorMatches, ".*already exists")
 	c.Assert(s.collection.CheckDuplicate(s.repo3), Equals, s.repo1)
 	c.Assert(s.collection.Add(s.repo4), IsNil)
+	c.Assert(s.collection.Add(s.repo5), IsNil)
 
-	r, err = s.collection.ByPrefixDistribution("ppa", "anaconda")
+	r, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, IsNil)
 
 	err = s.collection.LoadComplete(r, s.factory)
@@ -369,12 +465,15 @@ func (s *PublishedRepoCollectionSuite) TestAddByPrefixDistribution(c *C) {
 	c.Assert(r.String(), Equals, s.repo1.String())
 
 	collection := NewPublishedRepoCollection(s.db)
-	r, err = collection.ByPrefixDistribution("ppa", "anaconda")
+	r, err = collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, IsNil)
 
 	err = s.collection.LoadComplete(r, s.factory)
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, s.repo1.String())
+
+	r, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "precise")
+	c.Check(r.String(), Equals, s.repo5.String())
 }
 
 func (s *PublishedRepoCollectionSuite) TestByUUID(c *C) {
@@ -396,20 +495,63 @@ func (s *PublishedRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 	c.Assert(s.collection.Update(s.repo4), IsNil)
 
 	collection := NewPublishedRepoCollection(s.db)
-	r, err := collection.ByPrefixDistribution("ppa", "anaconda")
+	r, err := collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, IsNil)
-	c.Assert(r.snapshot, IsNil)
+	c.Assert(r.sourceItems["main"].snapshot, IsNil)
 	c.Assert(s.collection.LoadComplete(r, s.factory), IsNil)
-	c.Assert(r.snapshot.UUID, Equals, s.repo1.snapshot.UUID)
-	c.Assert(r.RefList().Len(), Equals, 0)
+	c.Assert(r.Sources["main"], Equals, s.repo1.sourceItems["main"].snapshot.UUID)
+	c.Assert(r.RefList("main").Len(), Equals, 0)
 
-	r, err = collection.ByPrefixDistribution("ppa", "precise")
+	r, err = collection.ByStoragePrefixDistribution("", "ppa", "precise")
 	c.Assert(err, IsNil)
-	c.Assert(r.localRepo, IsNil)
+	c.Assert(r.sourceItems["main"].localRepo, IsNil)
 	c.Assert(s.collection.LoadComplete(r, s.factory), IsNil)
-	c.Assert(r.localRepo.UUID, Equals, s.repo4.localRepo.UUID)
-	c.Assert(r.packageRefs.Len(), Equals, 0)
-	c.Assert(r.RefList().Len(), Equals, 0)
+	c.Assert(r.sourceItems["main"].localRepo.UUID, Equals, s.repo4.sourceItems["main"].localRepo.UUID)
+	c.Assert(r.sourceItems["main"].packageRefs.Len(), Equals, 0)
+	c.Assert(r.RefList("main").Len(), Equals, 0)
+}
+
+func (s *PublishedRepoCollectionSuite) TestLoadPre0_6(c *C) {
+	type oldPublishedRepo struct {
+		UUID          string
+		Prefix        string
+		Distribution  string
+		Origin        string
+		Label         string
+		Architectures []string
+		SourceKind    string
+		Component     string
+		SourceUUID    string `codec:"SnapshotUUID"`
+	}
+
+	old := oldPublishedRepo{
+		UUID:          s.repo1.UUID,
+		Prefix:        "ppa",
+		Distribution:  "anaconda",
+		Architectures: []string{"i386"},
+		SourceKind:    "local",
+		Component:     "contrib",
+		SourceUUID:    s.localRepo.UUID,
+	}
+
+	var buf bytes.Buffer
+
+	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
+	encoder.Encode(&old)
+
+	c.Assert(s.db.Put(s.repo1.Key(), buf.Bytes()), IsNil)
+	c.Assert(s.db.Put(s.repo1.RefKey(""), s.localRepo.RefList().Encode()), IsNil)
+
+	collection := NewPublishedRepoCollection(s.db)
+	repo, err := collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, IsNil)
+	c.Check(repo.Component, Equals, "")
+	c.Check(repo.SourceUUID, Equals, "")
+	c.Check(repo.Sources, DeepEquals, map[string]string{"contrib": s.localRepo.UUID})
+
+	c.Check(collection.LoadComplete(repo, s.factory), IsNil)
+	c.Check(repo.sourceItems["contrib"].localRepo.UUID, Equals, s.localRepo.UUID)
+	c.Check(repo.RefList("contrib").Len(), Equals, 0)
 }
 
 func (s *PublishedRepoCollectionSuite) TestForEachAndLen(c *C) {
@@ -437,27 +579,29 @@ func (s *PublishedRepoCollectionSuite) TestBySnapshot(c *C) {
 	c.Check(s.collection.Add(s.repo1), IsNil)
 	c.Check(s.collection.Add(s.repo2), IsNil)
 
-	c.Check(s.collection.BySnapshot(s.snap1), DeepEquals, []*PublishedRepo{s.repo1})
+	c.Check(s.collection.BySnapshot(s.snap1), DeepEquals, []*PublishedRepo{s.repo1, s.repo2})
 	c.Check(s.collection.BySnapshot(s.snap2), DeepEquals, []*PublishedRepo{s.repo2})
 }
 
 func (s *PublishedRepoCollectionSuite) TestByLocalRepo(c *C) {
 	c.Check(s.collection.Add(s.repo1), IsNil)
 	c.Check(s.collection.Add(s.repo4), IsNil)
+	c.Check(s.collection.Add(s.repo5), IsNil)
 
-	c.Check(s.collection.ByLocalRepo(s.localRepo), DeepEquals, []*PublishedRepo{s.repo4})
+	c.Check(s.collection.ByLocalRepo(s.localRepo), DeepEquals, []*PublishedRepo{s.repo4, s.repo5})
 }
 
 type PublishedRepoRemoveSuite struct {
 	PackageListMixinSuite
-	db                         database.Storage
-	factory                    *CollectionFactory
-	snapshotCollection         *SnapshotCollection
-	collection                 *PublishedRepoCollection
-	root                       string
-	publishedStorage           aptly.PublishedStorage
-	snap1                      *Snapshot
-	repo1, repo2, repo3, repo4 *PublishedRepo
+	db                                  database.Storage
+	factory                             *CollectionFactory
+	snapshotCollection                  *SnapshotCollection
+	collection                          *PublishedRepoCollection
+	root, root2                         string
+	provider                            *FakeStorageProvider
+	publishedStorage, publishedStorage2 *files.PublishedStorage
+	snap1                               *Snapshot
+	repo1, repo2, repo3, repo4, repo5   *PublishedRepo
 }
 
 var _ = Suite(&PublishedRepoRemoveSuite{})
@@ -472,16 +616,18 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 
 	s.snapshotCollection.Add(s.snap1)
 
-	s.repo1, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap1, s.factory)
-	s.repo2, _ = NewPublishedRepo("", "anaconda", "main", []string{}, s.snap1, s.factory)
-	s.repo3, _ = NewPublishedRepo("ppa", "meduza", "main", []string{}, s.snap1, s.factory)
-	s.repo4, _ = NewPublishedRepo("ppa", "osminog", "contrib", []string{}, s.snap1, s.factory)
+	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo3, _ = NewPublishedRepo("", "ppa", "meduza", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo4, _ = NewPublishedRepo("", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory)
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory)
 
 	s.collection = s.factory.PublishedRepoCollection()
 	s.collection.Add(s.repo1)
 	s.collection.Add(s.repo2)
 	s.collection.Add(s.repo3)
 	s.collection.Add(s.repo4)
+	s.collection.Add(s.repo5)
 
 	s.root = c.MkDir()
 	s.publishedStorage = files.NewPublishedStorage(s.root)
@@ -492,6 +638,15 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 	s.publishedStorage.MkDir("ppa/pool/contrib")
 	s.publishedStorage.MkDir("dists/anaconda")
 	s.publishedStorage.MkDir("pool/main")
+
+	s.root2 = c.MkDir()
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.publishedStorage2.MkDir("ppa/dists/osminog")
+	s.publishedStorage2.MkDir("ppa/pool/contrib")
+
+	s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{
+		"":            s.publishedStorage,
+		"files:other": s.publishedStorage2}}
 }
 
 func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
@@ -499,7 +654,7 @@ func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
-	s.repo1.RemoveFiles(s.publishedStorage, false, false, nil)
+	s.repo1.RemoveFiles(s.provider, false, []string{}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
@@ -508,10 +663,12 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
-	s.repo1.RemoveFiles(s.publishedStorage, false, true, nil)
+	s.repo1.RemoveFiles(s.provider, false, []string{"main"}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
@@ -520,10 +677,26 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithTwoPools(c *C) {
+	s.repo1.RemoveFiles(s.provider, false, []string{"main", "contrib"}, nil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
-	s.repo1.RemoveFiles(s.publishedStorage, true, true, nil)
+	s.repo1.RemoveFiles(s.provider, true, []string{"main"}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
@@ -532,10 +705,12 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
-	s.repo2.RemoveFiles(s.publishedStorage, true, true, nil)
+	s.repo2.RemoveFiles(s.provider, true, []string{"main"}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
@@ -543,17 +718,19 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
-	err := s.collection.Remove(s.publishedStorage, "ppa", "anaconda", s.factory, nil)
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
 	c.Check(err, IsNil)
 
-	_, err = s.collection.ByPrefixDistribution("ppa", "anaconda")
+	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Check(err, ErrorMatches, ".*not found")
 
 	collection := NewPublishedRepoCollection(s.db)
-	_, err = collection.ByPrefixDistribution("ppa", "anaconda")
+	_, err = collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Check(err, ErrorMatches, ".*not found")
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
@@ -563,11 +740,13 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 
-	err = s.collection.Remove(s.publishedStorage, "ppa", "anaconda", s.factory, nil)
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
 	c.Check(err, ErrorMatches, ".*not found")
 
-	err = s.collection.Remove(s.publishedStorage, "ppa", "meduza", s.factory, nil)
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil)
 	c.Check(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
@@ -577,17 +756,19 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
-	err := s.collection.Remove(s.publishedStorage, ".", "anaconda", s.factory, nil)
+	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil)
 	c.Check(err, IsNil)
 
-	_, err = s.collection.ByPrefixDistribution(".", "anaconda")
+	_, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda")
 	c.Check(err, ErrorMatches, ".*not found")
 
 	collection := NewPublishedRepoCollection(s.db)
-	_, err = collection.ByPrefixDistribution(".", "anaconda")
+	_, err = collection.ByStoragePrefixDistribution("", ".", "anaconda")
 	c.Check(err, ErrorMatches, ".*not found")
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
@@ -597,4 +778,28 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) {
+	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	collection := NewPublishedRepoCollection(s.db)
+	_, err = collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
 }

deb/query.go

@@ -0,0 +1,262 @@
+package deb
+
+import (
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// PackageQuery is interface of predicate on Package
+type PackageQuery interface {
+	// Matches calculates match of condition against package
+	Matches(pkg *Package) bool
+	// Fast returns if search strategy is possible for this query
+	Fast() bool
+	// Query performs search on package list
+	Query(list *PackageList) *PackageList
+	// String interface
+	String() string
+}
+
+// OrQuery is L | R
+type OrQuery struct {
+	L, R PackageQuery
+}
+
+// AndQuery is L , R
+type AndQuery struct {
+	L, R PackageQuery
+}
+
+// NotQuery is ! Q
+type NotQuery struct {
+	Q PackageQuery
+}
+
+// FieldQuery is generic request against field
+type FieldQuery struct {
+	Field    string
+	Relation int
+	Value    string
+	Regexp   *regexp.Regexp `codec:"-"`
+}
+
+// PkgQuery is search request against specific package
+type PkgQuery struct {
+	Pkg     string
+	Version string
+	Arch    string
+}
+
+// DependencyQuery is generic Debian-dependency like query
+type DependencyQuery struct {
+	Dep Dependency
+}
+
+// Matches if any of L, R matches
+func (q *OrQuery) Matches(pkg *Package) bool {
+	return q.L.Matches(pkg) || q.R.Matches(pkg)
+}
+
+// Fast is true only if both parts are fast
+func (q *OrQuery) Fast() bool {
+	return q.L.Fast() && q.R.Fast()
+}
+
+// Query strategy depends on nodes
+func (q *OrQuery) Query(list *PackageList) (result *PackageList) {
+	if q.Fast() {
+		result = q.L.Query(list)
+		result.Append(q.R.Query(list))
+	} else {
+		result = list.Scan(q)
+	}
+	return
+}
+
+// String interface
+func (q *OrQuery) String() string {
+	return fmt.Sprintf("(%s) | (%s)", q.L, q.R)
+}
+
+// Matches if both of L, R matches
+func (q *AndQuery) Matches(pkg *Package) bool {
+	return q.L.Matches(pkg) && q.R.Matches(pkg)
+}
+
+// Fast is true if any of the parts are fast
+func (q *AndQuery) Fast() bool {
+	return q.L.Fast() || q.R.Fast()
+}
+
+// Query strategy depends on nodes
+func (q *AndQuery) Query(list *PackageList) (result *PackageList) {
+	if !q.Fast() {
+		result = list.Scan(q)
+	} else {
+		if q.L.Fast() {
+			result = q.L.Query(list)
+			result = result.Scan(q.R)
+		} else {
+			result = q.R.Query(list)
+			result = result.Scan(q.L)
+		}
+	}
+	return
+}
+
+// String interface
+func (q *AndQuery) String() string {
+	return fmt.Sprintf("(%s), (%s)", q.L, q.R)
+}
+
+// Matches if not matches
+func (q *NotQuery) Matches(pkg *Package) bool {
+	return !q.Q.Matches(pkg)
+}
+
+// Fast is false
+func (q *NotQuery) Fast() bool {
+	return false
+}
+
+// Query strategy is scan always
+func (q *NotQuery) Query(list *PackageList) (result *PackageList) {
+	result = list.Scan(q)
+	return
+}
+
+// String interface
+func (q *NotQuery) String() string {
+	return fmt.Sprintf("!(%s)", q.Q)
+}
+
+// Matches on generic field
+func (q *FieldQuery) Matches(pkg *Package) bool {
+	if q.Field == "$Version" {
+		return pkg.MatchesDependency(Dependency{Pkg: pkg.Name, Relation: q.Relation, Version: q.Value, Regexp: q.Regexp})
+	}
+	if q.Field == "$Architecture" && q.Relation == VersionEqual {
+		return pkg.MatchesArchitecture(q.Value)
+	}
+
+	field := pkg.GetField(q.Field)
+
+	switch q.Relation {
+	case VersionDontCare:
+		return field != ""
+	case VersionEqual:
+		return field == q.Value
+	case VersionGreater:
+		return field > q.Value
+	case VersionGreaterOrEqual:
+		return field >= q.Value
+	case VersionLess:
+		return field < q.Value
+	case VersionLessOrEqual:
+		return field <= q.Value
+	case VersionPatternMatch:
+		matched, err := filepath.Match(q.Value, field)
+		return err == nil && matched
+	case VersionRegexp:
+		if q.Regexp == nil {
+			q.Regexp = regexp.MustCompile(q.Value)
+		}
+		return q.Regexp.FindStringIndex(field) != nil
+
+	}
+	panic("unknown relation")
+}
+
+// Query runs iteration through list
+func (q *FieldQuery) Query(list *PackageList) (result *PackageList) {
+	result = list.Scan(q)
+	return
+}
+
+// Fast depends on the query
+func (q *FieldQuery) Fast() bool {
+	return false
+}
+
+// String interface
+func (q *FieldQuery) String() string {
+	escape := func(val string) string {
+		if strings.IndexAny(val, "()|,!{} \t\n") != -1 {
+			return "'" + strings.Replace(strings.Replace(val, "\\", "\\\\", -1), "'", "\\'", -1) + "'"
+		}
+		return val
+	}
+
+	var op string
+	switch q.Relation {
+	case VersionEqual:
+		op = "="
+	case VersionGreater:
+		op = ">>"
+	case VersionLess:
+		op = "<<"
+	case VersionRegexp:
+		op = "~"
+	case VersionPatternMatch:
+		op = "%"
+	case VersionGreaterOrEqual:
+		op = ">="
+	case VersionLessOrEqual:
+		op = "<="
+	}
+	return fmt.Sprintf("%s (%s %s)", escape(q.Field), op, escape(q.Value))
+}
+
+// Matches on dependency condition
+func (q *DependencyQuery) Matches(pkg *Package) bool {
+	return pkg.MatchesDependency(q.Dep)
+}
+
+// Fast is always true for dependency query
+func (q *DependencyQuery) Fast() bool {
+	return true
+}
+
+// Query runs PackageList.Search
+func (q *DependencyQuery) Query(list *PackageList) (result *PackageList) {
+	result = NewPackageList()
+	for _, pkg := range list.Search(q.Dep, true) {
+		result.Add(pkg)
+	}
+
+	return
+}
+
+// String interface
+func (q *DependencyQuery) String() string {
+	return q.Dep.String()
+}
+
+// Matches on specific properties
+func (q *PkgQuery) Matches(pkg *Package) bool {
+	return pkg.Name == q.Pkg && pkg.Version == q.Version && pkg.Architecture == q.Arch
+}
+
+// Fast is always true for package query
+func (q *PkgQuery) Fast() bool {
+	return true
+}
+
+// Query looks up specific package
+func (q *PkgQuery) Query(list *PackageList) (result *PackageList) {
+	result = NewPackageList()
+
+	pkg := list.packages["P"+q.Arch+" "+q.Pkg+" "+q.Version]
+	if pkg != nil {
+		result.Add(pkg)
+	}
+
+	return
+}
+
+// String interface
+func (q *PkgQuery) String() string {
+	return fmt.Sprintf("%s_%s_%s", q.Pkg, q.Version, q.Arch)
+}

deb/reflist.go

@@ -194,25 +194,28 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle
 				}
 			}
 
-			// is pl & pr the same package, but different version?
-			if pl.Name == pr.Name && pl.Architecture == pr.Architecture {
-				result = append(result, PackageDiff{Left: pl, Right: pr})
-				il++
-				ir++
-				pl, pr = nil, nil
-			} else {
-				// otherwise pl or pr is missing on one of the sides
-				if rel < 0 {
+			// otherwise pl or pr is missing on one of the sides
+			if rel < 0 {
+				// compaction: +(,A) -(B,) --> !(A,B)
+				if len(result) > 0 && result[len(result)-1].Left == nil && result[len(result)-1].Right.Name == pl.Name &&
+					result[len(result)-1].Right.Architecture == pl.Architecture {
+					result[len(result)-1] = PackageDiff{Left: pl, Right: result[len(result)-1].Right}
+				} else {
 					result = append(result, PackageDiff{Left: pl, Right: nil})
-					il++
-					pl = nil
+				}
+				il++
+				pl = nil
+			} else {
+				// compaction: -(A,) +(,B) --> !(A,B)
+				if len(result) > 0 && result[len(result)-1].Right == nil && result[len(result)-1].Left.Name == pr.Name &&
+					result[len(result)-1].Left.Architecture == pr.Architecture {
+					result[len(result)-1] = PackageDiff{Left: result[len(result)-1].Left, Right: pr}
 				} else {
 					result = append(result, PackageDiff{Left: nil, Right: pr})
-					ir++
-					pr = nil
 				}
+				ir++
+				pr = nil
 			}
-
 		}
 	}
 

deb/reflist_test.go

@@ -31,7 +31,7 @@ func (s *PackageRefListSuite) SetUpTest(c *C) {
 	stanza["Package"] = "mars-invaders"
 	s.p3 = NewPackageFromControlFile(stanza)
 	stanza = packageStanza.Copy()
-	stanza["Size"] = "42"
+	stanza["Source"] = "unknown-planet"
 	s.p4 = NewPackageFromControlFile(stanza)
 	stanza = packageStanza.Copy()
 	stanza["Package"] = "lonely-strangers"

deb/remote.go

@@ -43,6 +43,10 @@ type RemoteRepo struct {
 	LastDownloadDate time.Time
 	// Checksums for release files
 	ReleaseFiles map[string]utils.ChecksumInfo
+	// Filter for packages
+	Filter string
+	// FilterWithDeps to include dependencies from filter query
+	FilterWithDeps bool
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 	// Parsed archived root
@@ -320,7 +324,8 @@ ok:
 }
 
 // Download downloads all repo files
-func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory, packagePool aptly.PackagePool, ignoreMismatch bool) error {
+func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
+	packagePool aptly.PackagePool, ignoreMismatch bool, dependencyOptions int, filterQuery PackageQuery) error {
 	list := NewPackageList()
 
 	progress.Printf("Downloading & parsing package files...\n")
@@ -393,6 +398,25 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, co
 		progress.ShutdownBar()
 	}
 
+	var err error
+
+	if repo.Filter != "" {
+		progress.Printf("Applying filter...\n")
+
+		list.PrepareIndex()
+
+		emptyList := NewPackageList()
+		emptyList.PrepareIndex()
+
+		origPackages := list.Len()
+		list, err = list.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures)
+		if err != nil {
+			return err
+		}
+
+		progress.Printf("Packages filtered: %d -> %d.\n", origPackages, list.Len())
+	}
+
 	progress.Printf("Building download queue...\n")
 
 	// Build download queue
@@ -400,7 +424,7 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, co
 	count := 0
 	downloadSize := int64(0)
 
-	err := list.ForEach(func(p *Package) error {
+	err = list.ForEach(func(p *Package) error {
 		list, err2 := p.DownloadList(packagePool)
 		if err2 != nil {
 			return err2
@@ -478,7 +502,43 @@ func (repo *RemoteRepo) Decode(input []byte) error {
 	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
 	err := decoder.Decode(repo)
 	if err != nil {
-		return err
+		if strings.HasPrefix(err.Error(), "codec.decoder: readContainerLen: Unrecognized descriptor byte: hex: 80") {
+			// probably it is broken DB from go < 1.2, try decoding w/o time.Time
+			var repo11 struct {
+				UUID             string
+				Name             string
+				ArchiveRoot      string
+				Distribution     string
+				Components       []string
+				Architectures    []string
+				DownloadSources  bool
+				Meta             Stanza
+				LastDownloadDate []byte
+				ReleaseFiles     map[string]utils.ChecksumInfo
+				Filter           string
+				FilterWithDeps   bool
+			}
+
+			decoder = codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
+			err2 := decoder.Decode(&repo11)
+			if err2 != nil {
+				return err
+			}
+
+			repo.UUID = repo11.UUID
+			repo.Name = repo11.Name
+			repo.ArchiveRoot = repo11.ArchiveRoot
+			repo.Distribution = repo11.Distribution
+			repo.Components = repo11.Components
+			repo.Architectures = repo11.Architectures
+			repo.DownloadSources = repo11.DownloadSources
+			repo.Meta = repo11.Meta
+			repo.ReleaseFiles = repo11.ReleaseFiles
+			repo.Filter = repo11.Filter
+			repo.FilterWithDeps = repo11.FilterWithDeps
+		} else {
+			return err
+		}
 	}
 	return repo.prepare()
 }

deb/remote_test.go

@@ -251,7 +251,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
 
-	err = s.repo.Download(s.progress, s.downloader, s.collectionFactory, s.packagePool, false)
+	err = s.repo.Download(s.progress, s.downloader, s.collectionFactory, s.packagePool, false, 0, nil)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 	c.Assert(s.repo.packageRefs, NotNil)
@@ -284,7 +284,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz", "abcd")
 	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz", "abcde")
 
-	err = s.repo.Download(s.progress, s.downloader, s.collectionFactory, s.packagePool, false)
+	err = s.repo.Download(s.progress, s.downloader, s.collectionFactory, s.packagePool, false, 0, nil)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 	c.Assert(s.repo.packageRefs, NotNil)
@@ -319,7 +319,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.Download(s.progress, downloader, s.collectionFactory, s.packagePool, false)
+	err = s.flat.Download(s.progress, downloader, s.collectionFactory, s.packagePool, false, 0, nil)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 	c.Assert(s.flat.packageRefs, NotNil)
@@ -353,7 +353,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.Download(s.progress, downloader, s.collectionFactory, s.packagePool, false)
+	err = s.flat.Download(s.progress, downloader, s.collectionFactory, s.packagePool, false, 0, nil)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 	c.Assert(s.flat.packageRefs, NotNil)

deb/snapshot.go

@@ -9,6 +9,7 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/ugorji/go/codec"
 	"log"
+	"strings"
 	"time"
 )
 
@@ -125,7 +126,36 @@ func (s *Snapshot) Encode() []byte {
 // Decode decodes msgpack representation into Snapshot
 func (s *Snapshot) Decode(input []byte) error {
 	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
-	return decoder.Decode(s)
+	err := decoder.Decode(s)
+	if err != nil {
+		if strings.HasPrefix(err.Error(), "codec.decoder: readContainerLen: Unrecognized descriptor byte: hex: 80") {
+			// probably it is broken DB from go < 1.2, try decoding w/o time.Time
+			var snapshot11 struct {
+				UUID string
+				Name string
+				CreatedAt []byte
+
+				SourceKind string
+				SourceIDs  []string
+				Description string
+			}
+
+			decoder = codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
+			err2 := decoder.Decode(&snapshot11)
+			if err2 != nil {
+				return err
+			}
+
+			s.UUID = snapshot11.UUID
+			s.Name = snapshot11.Name
+			s.SourceKind = snapshot11.SourceKind
+			s.SourceIDs = snapshot11.SourceIDs
+			s.Description = snapshot11.Description
+		} else {
+			return err
+		}
+	}
+	return nil
 }
 
 // SnapshotCollection does listing, updating/adding/deleting of Snapshots
@@ -178,7 +208,10 @@ func (collection *SnapshotCollection) Update(snapshot *Snapshot) error {
 	if err != nil {
 		return err
 	}
-	return collection.db.Put(snapshot.RefKey(), snapshot.packageRefs.Encode())
+	if snapshot.packageRefs != nil {
+		return collection.db.Put(snapshot.RefKey(), snapshot.packageRefs.Encode())
+	}
+	return nil
 }
 
 // LoadComplete loads additional information about snapshot

deb/version.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 	"unicode"
@@ -178,6 +179,8 @@ const (
 	VersionEqual
 	VersionGreaterOrEqual
 	VersionGreater
+	VersionPatternMatch
+	VersionRegexp
 )
 
 // Dependency is a parsed version of Debian dependency to package
@@ -186,6 +189,7 @@ type Dependency struct {
 	Relation     int
 	Version      string
 	Architecture string
+	Regexp       *regexp.Regexp
 }
 
 // Hash calculates some predefined unique ID of Dependency
@@ -207,6 +211,10 @@ func (d *Dependency) String() string {
 		rel = ">="
 	case VersionLessOrEqual:
 		rel = "<="
+	case VersionPatternMatch:
+		rel = "%"
+	case VersionRegexp:
+		rel = "~"
 	case VersionDontCare:
 		return fmt.Sprintf("%s [%s]", d.Pkg, d.Architecture)
 	}

files/public.go

@@ -1,10 +1,12 @@
 package files
 
 import (
+	"fmt"
 	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/utils"
+	"io"
 	"os"
 	"path/filepath"
+	"syscall"
 )
 
 // PublishedStorage abstract file system with public dirs (published repos)
@@ -12,9 +14,10 @@ type PublishedStorage struct {
 	rootPath string
 }
 
-// Check interface
+// Check interfaces
 var (
-	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+	_ aptly.PublishedStorage      = (*PublishedStorage)(nil)
+	_ aptly.LocalPublishedStorage = (*PublishedStorage)(nil)
 )
 
 // NewPublishedStorage creates new instance of PublishedStorage which specified root
@@ -32,9 +35,26 @@ func (storage *PublishedStorage) MkDir(path string) error {
 	return os.MkdirAll(filepath.Join(storage.rootPath, path), 0755)
 }
 
-// CreateFile creates file for writing under public path
-func (storage *PublishedStorage) CreateFile(path string) (*os.File, error) {
-	return os.Create(filepath.Join(storage.rootPath, path))
+// PutFile puts file into published storage at specified path
+func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
+	var (
+		source, f *os.File
+		err       error
+	)
+	source, err = os.Open(sourceFilename)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	f, err = os.Create(filepath.Join(storage.rootPath, path))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = io.Copy(f, source)
+	return err
 }
 
 // Remove removes single file under public path
@@ -59,7 +79,8 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 // sourcePath is filepath to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool, sourcePath string) error {
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool,
+	sourcePath, sourceMD5 string, force bool) error {
 	// verify that package pool is local pool is filesystem pool
 	_ = sourcePool.(*PackagePool)
 
@@ -71,11 +92,38 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 		return err
 	}
 
-	_, err = os.Stat(filepath.Join(poolPath, baseName))
-	if err == nil { // already exists, skip
-		return nil
+	var dstStat, srcStat os.FileInfo
+
+	dstStat, err = os.Stat(filepath.Join(poolPath, baseName))
+	if err == nil {
+		// already exists, check source file
+		srcStat, err = os.Stat(sourcePath)
+		if err != nil {
+			// source file doesn't exist? problem!
+			return err
+		}
+
+		srcSys := srcStat.Sys().(*syscall.Stat_t)
+		dstSys := dstStat.Sys().(*syscall.Stat_t)
+
+		// source and destination inodes match, no need to link
+		if srcSys.Ino == dstSys.Ino {
+			return nil
+		}
+
+		// source and destination have different inodes, if !forced, this is fatal error
+		if !force {
+			return fmt.Errorf("error linking file to %s: file already exists and is different", filepath.Join(poolPath, baseName))
+		}
+
+		// forced, so remove destination
+		err = os.Remove(filepath.Join(poolPath, baseName))
+		if err != nil {
+			return err
+		}
 	}
 
+	// destination doesn't exist (or forced), create link
 	return os.Link(sourcePath, filepath.Join(poolPath, baseName))
 }
 
@@ -94,12 +142,12 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 		return nil
 	})
 
-	return result, err
-}
+	if err != nil && os.IsNotExist(err) {
+		// file path doesn't exist, consider it empty
+		return []string{}, nil
+	}
 
-// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
-func (storage *PublishedStorage) ChecksumsForFile(path string) (utils.ChecksumInfo, error) {
-	return utils.ChecksumsForFile(filepath.Join(storage.rootPath, path))
+	return result, err
 }
 
 // RenameFile renames (moves) file

files/public_test.go

@@ -32,13 +32,12 @@ func (s *PublishedStorageSuite) TestMkDir(c *C) {
 	c.Assert(err, IsNil)
 }
 
-func (s *PublishedStorageSuite) TestCreateFile(c *C) {
+func (s *PublishedStorageSuite) TesPutFile(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
-	file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file.Close()
 
 	_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
 	c.Assert(err, IsNil)
@@ -48,26 +47,27 @@ func (s *PublishedStorageSuite) TestFilelist(c *C) {
 	err := s.storage.MkDir("ppa/pool/main/a/ab/")
 	c.Assert(err, IsNil)
 
-	file, err := s.storage.CreateFile("ppa/pool/main/a/ab/a.deb")
+	err = s.storage.PutFile("ppa/pool/main/a/ab/a.deb", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file.Close()
 
-	file2, err := s.storage.CreateFile("ppa/pool/main/a/ab/b.deb")
+	err = s.storage.PutFile("ppa/pool/main/a/ab/b.deb", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file2.Close()
 
 	list, err := s.storage.Filelist("ppa/pool/main/")
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []string{"a/ab/a.deb", "a/ab/b.deb"})
+
+	list, err = s.storage.Filelist("ppa/pool/doenstexist/")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
 }
 
 func (s *PublishedStorageSuite) TestRenameFile(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
-	file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file.Close()
 
 	err = s.storage.RenameFile("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease")
 	c.Check(err, IsNil)
@@ -80,9 +80,8 @@ func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
-	file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file.Close()
 
 	err = s.storage.RemoveDirs("ppa/dists/", nil)
 
@@ -95,9 +94,8 @@ func (s *PublishedStorageSuite) TestRemove(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
-	file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file.Close()
 
 	err = s.storage.Remove("ppa/dists/squeeze/Release")
 
@@ -155,7 +153,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		err = ioutil.WriteFile(t.sourcePath, []byte("Contents"), 0644)
 		c.Assert(err, IsNil)
 
-		err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath)
+		err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, "", false)
 		c.Assert(err, IsNil)
 
 		st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename))
@@ -164,4 +162,31 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		info := st.Sys().(*syscall.Stat_t)
 		c.Check(int(info.Nlink), Equals, 2)
 	}
+
+	// test linking files to duplicate final name
+	sourcePath := filepath.Join(s.root, "pool/02/bc/mars-invaders_1.03.deb")
+	err := os.MkdirAll(filepath.Dir(sourcePath), 0755)
+	c.Assert(err, IsNil)
+
+	err = ioutil.WriteFile(sourcePath, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "", false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different")
+
+	st, err := os.Stat(sourcePath)
+	c.Assert(err, IsNil)
+
+	info := st.Sys().(*syscall.Stat_t)
+	c.Check(int(info.Nlink), Equals, 1)
+
+	// linking with force
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "", true)
+	c.Check(err, IsNil)
+
+	st, err = os.Stat(sourcePath)
+	c.Assert(err, IsNil)
+
+	info = st.Sys().(*syscall.Stat_t)
+	c.Check(int(info.Nlink), Equals, 2)
 }

http/download.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"code.google.com/p/mxk/go1/flowcontrol"
 	"compress/bzip2"
 	"compress/gzip"
 	"fmt"
@@ -21,14 +22,15 @@ var (
 
 // downloaderImpl is implementation of Downloader interface
 type downloaderImpl struct {
-	queue    chan *downloadTask
-	stop     chan bool
-	stopped  chan bool
-	pause    chan bool
-	unpause  chan bool
-	progress aptly.Progress
-	threads  int
-	client   *http.Client
+	queue     chan *downloadTask
+	stop      chan struct{}
+	stopped   chan struct{}
+	pause     chan struct{}
+	unpause   chan struct{}
+	progress  aptly.Progress
+	aggWriter io.Writer
+	threads   int
+	client    *http.Client
 }
 
 // downloadTask represents single item in queue
@@ -41,24 +43,30 @@ type downloadTask struct {
 }
 
 // NewDownloader creates new instance of Downloader which specified number
-// of threads
-func NewDownloader(threads int, progress aptly.Progress) aptly.Downloader {
+// of threads and download limit in bytes/sec
+func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.Downloader {
+	transport := *http.DefaultTransport.(*http.Transport)
+	transport.DisableCompression = true
+
 	downloader := &downloaderImpl{
 		queue:    make(chan *downloadTask, 1000),
-		stop:     make(chan bool),
-		stopped:  make(chan bool),
-		pause:    make(chan bool),
-		unpause:  make(chan bool),
+		stop:     make(chan struct{}),
+		stopped:  make(chan struct{}),
+		pause:    make(chan struct{}),
+		unpause:  make(chan struct{}),
 		threads:  threads,
 		progress: progress,
 		client: &http.Client{
-			Transport: &http.Transport{
-				DisableCompression: true,
-				Proxy:              http.ProxyFromEnvironment,
-			},
+			Transport: &transport,
 		},
 	}
 
+	if downLimit > 0 {
+		downloader.aggWriter = flowcontrol.NewWriter(progress, downLimit)
+	} else {
+		downloader.aggWriter = progress
+	}
+
 	for i := 0; i < downloader.threads; i++ {
 		go downloader.process()
 	}
@@ -70,7 +78,7 @@ func NewDownloader(threads int, progress aptly.Progress) aptly.Downloader {
 // but doesn't process rest of queue
 func (downloader *downloaderImpl) Shutdown() {
 	for i := 0; i < downloader.threads; i++ {
-		downloader.stop <- true
+		downloader.stop <- struct{}{}
 	}
 
 	for i := 0; i < downloader.threads; i++ {
@@ -81,14 +89,14 @@ func (downloader *downloaderImpl) Shutdown() {
 // Pause pauses task processing
 func (downloader *downloaderImpl) Pause() {
 	for i := 0; i < downloader.threads; i++ {
-		downloader.pause <- true
+		downloader.pause <- struct{}{}
 	}
 }
 
 // Resume resumes task processing
 func (downloader *downloaderImpl) Resume() {
 	for i := 0; i < downloader.threads; i++ {
-		downloader.unpause <- true
+		downloader.unpause <- struct{}{}
 	}
 }
 
@@ -140,7 +148,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	defer outfile.Close()
 
 	checksummer := utils.NewChecksumWriter()
-	writers := []io.Writer{outfile, downloader.progress}
+	writers := []io.Writer{outfile, downloader.aggWriter}
 
 	if task.expected.Size != -1 {
 		writers = append(writers, checksummer)
@@ -194,7 +202,7 @@ func (downloader *downloaderImpl) process() {
 	for {
 		select {
 		case <-downloader.stop:
-			downloader.stopped <- true
+			downloader.stopped <- struct{}{}
 			return
 		case <-downloader.pause:
 			<-downloader.unpause

http/download_test.go

@@ -60,7 +60,7 @@ func (s *DownloaderSuite) TearDownTest(c *C) {
 func (s *DownloaderSuite) TestStartupShutdown(c *C) {
 	goroutines := runtime.NumGoroutine()
 
-	d := NewDownloader(10, s.progress)
+	d := NewDownloader(10, 100, s.progress)
 	d.Shutdown()
 
 	// wait for goroutines to shutdown
@@ -72,7 +72,7 @@ func (s *DownloaderSuite) TestStartupShutdown(c *C) {
 }
 
 func (s *DownloaderSuite) TestPauseResume(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 
 	d.Pause()
@@ -80,7 +80,7 @@ func (s *DownloaderSuite) TestPauseResume(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadOK(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -90,7 +90,7 @@ func (s *DownloaderSuite) TestDownloadOK(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -131,7 +131,7 @@ func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownload404(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -141,7 +141,7 @@ func (s *DownloaderSuite) TestDownload404(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadConnectError(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -151,7 +151,7 @@ func (s *DownloaderSuite) TestDownloadConnectError(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadFileError(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -161,7 +161,7 @@ func (s *DownloaderSuite) TestDownloadFileError(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadTemp(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 
 	f, err := DownloadTemp(d, s.url+"/test")
@@ -178,7 +178,7 @@ func (s *DownloaderSuite) TestDownloadTemp(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadTempWithChecksum(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 
 	f, err := DownloadTempWithChecksum(d, s.url+"/test", utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
@@ -191,7 +191,7 @@ func (s *DownloaderSuite) TestDownloadTempWithChecksum(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadTempError(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 
 	f, err := DownloadTemp(d, s.url+"/doesntexist")

man/aptly.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "APTLY" "1" "May 2014" "" ""
+.TH "APTLY" "1" "August 2014" "" ""
 .
 .SH "NAME"
 \fBaptly\fR \- Debian repository management tool
@@ -37,6 +37,7 @@ Configuration file is stored in JSON format (default values shown below):
 {
   "rootDir": "$HOME/\.aptly",
   "downloadConcurrency": 4,
+  "downloadSpeedLimit": 0,
   "architectures": [],
   "dependencyFollowSuggests": false,
   "dependencyFollowRecommends": false
@@ -46,7 +47,16 @@ Configuration file is stored in JSON format (default values shown below):
   "gpgDisableVerify": false,
   "downloadSourcePackages": false,
   "ppaDistributorID": "ubuntu",
-  "ppaCodename": ""
+  "ppaCodename": "",
+  "S3PublishEndpoints": {
+    "test": {
+      "region": "us\-east\-1",
+      "bucket": "repo",
+      "awsAccessKeyID": ""
+      "awsSecretAccessKey": "",
+      "prefix": "",
+      "acl": "public\-read"
+    }
 }
 .
 .fi
@@ -65,6 +75,10 @@ is root of directory storage to store database (\fBrootDir\fR/db), downloaded pa
 is a number of parallel download threads to use when downloading packages
 .
 .TP
+\fBdownloadSpeedLimit\fR
+limit in kbytes/sec on download speed while mirroring remote repositieis
+.
+.TP
 \fBarchitectures\fR
 is a list of architectures to process; if left empty defaults to all available architectures; could be overridden with option \fB\-architectures\fR
 .
@@ -100,8 +114,41 @@ if enabled, all mirrors created would have flag set to download source packages;
 \fBppaDistributorID\fR, \fBppaCodename\fR
 specifies paramaters for short PPA url expansion, if left blank they default to output of \fBlsb_release\fR command
 .
-.SH "PACKAGE SPEC"
-Some commands accept package specs to identify list of packages to process\. Package spec is a list of following search conditions:
+.TP
+\fBS3PublisEndpoints\fR
+configuration of Amazon S3 publishing endpoints (see below)
+.
+.SH "S3 PUBLISHING ENDPOINTS"
+aptly could be configured to publish repository directly to Amazon S3\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings:
+.
+.TP
+\fBregion\fR
+Amazon region for S3 bucket (e\.g\. \fBus\-east\-1\fR)
+.
+.TP
+\fBbucket\fR
+bucket name
+.
+.TP
+\fBprefix\fR
+(optional) do publishing under specified prefix in the bucket, defaults to no prefix (bucket root)
+.
+.TP
+\fBacl\fR
+(optional) assign ACL to published files (one of the canned ACLs in Amazon terminology)\. Useful values: \fBprivate\fR (default) or \fBpublic\-read\fR (public repository)\. Public repositories could be consumed by \fBapt\fR using HTTP endpoint (Amazon bucket should be configured for "website hosting"), for private repositories special apt S3 transport is required\.
+.
+.TP
+\fBawsAccessKeyID\fR, \fBawsSecretAccessKey\fR
+(optional) Amazon credentials to access S3 bucket\. If not supplied, environment variables \fBAWS_ACCESS_KEY_ID\fR and \fBAWS_SECRET_ACCESS_KEY\fR are used\.
+.
+.P
+In order to publish to S3, specify endpoint as \fBs3:endpoint\-name:\fR before publishing prefix on the command line, e\.g\.:
+.
+.P
+\fBaptly publish snapshot wheezy\-main s3:test:\fR
+.
+.SH "PACKAGE QUERY"
+Some commands accept package queries to identify list of packages to process\. Package query syntax almost matches \fBreprepro\fR query language\. Query consists of the following simple terms:
 .
 .TP
 direct package reference
@@ -109,14 +156,63 @@ reference to exaclty one package\. Format is identical to the way aptly lists pa
 .
 .TP
 dependency condition
-syntax follows Debian dependency specification: package_name followed by optional version specification and architecture limit\.
+syntax follows Debian dependency specification: package_name followed by optional version specification and architecture limit, e\.g: \fBmysql\-client (>= 3\.6)\fR\.
+.
+.TP
+query against package fields
+syntax is the same as for dependency conditions, but instead of package name field name is used, e\.g: \fBPriority (optional)\fR\.
+.
+.P
+Supported fields:
+.
+.IP "\[ci]" 4
+all field names from Debian package control files are supported except for \fBFilename\fR, \fBMD5sum\fR, \fBSHA1\fR, \fBSHA256\fR, \fBSize\fR, \fBFiles\fR, \fBChecksums\-SHA1\fR, \fBChecksums\-SHA256\fR\.
+.
+.IP "\[ci]" 4
+\fB$Source\fR is a name of source package (for binary packages)
+.
+.IP "\[ci]" 4
+\fB$SourceVersion\fR is a version of source package
+.
+.IP "\[ci]" 4
+\fB$Architecture\fR is \fBArchitecture\fR for binary packages and \fBsource\fR for source packages, when matching with equal (\fB=\fR) operator, package with \fBany\fR architecture matches all architectures but \fBsource\fR\.
+.
+.IP "\[ci]" 4
+\fB$Version\fR has the same value as \fBVersion\fR, but comparison operators use Debian version precedence rules
+.
+.IP "\[ci]" 4
+\fB$PackageType\fR is \fBdeb\fR for binary packages and \fBsource\fR for source packages
+.
+.IP "" 0
+.
+.P
+Operators:
+.
+.TP
+\fB=\fR
+strict match, default operator is no operator is given
+.
+.TP
+\fB>=\fR, \fB<=\fR, \fB=\fR, \fB>>\fR (strictly greater), \fB<<\fR (strictly less)
+lexicographical comparison for all fields and special rules when comparing package versions
+.
+.TP
+\fB%\fR
+pattern matching, like shell patterns, supported special symbols are: \fB[^]?*\fR, e\.g\.: \fB$Version (% 3\.5\-*)\fR
+.
+.TP
+\fB~\fR
+regular expression matching, e\.g\.: \fBName (~ \.*\-dev)\fR
+.
+.P
+Simple terms could be combined into more complex queries using operators \fB,\fR (and), \fB|\fR (or) and \fB!\fR (not), parentheses \fB()\fR are used to change operator precedence\. Match value could be enclosed in single (\fB\(cq\fR) or double (\fB"\fR) quotes if required to resolve ambiguity, quotes inside quoted string should escaped with slash (\fB\e\fR)\.
 .
 .P
 Examples:
 .
 .TP
 \fBmysql\-client\fR
-matches package mysql\-client of any version and architecture (including source)
+matches package mysql\-client of any version and architecture (including source), also matches packages that \fBProvide:\fR \fBmysql\-client\fR\.
 .
 .TP
 \fBmysql\-client (>= 3\.6)\fR
@@ -130,8 +226,20 @@ matches package \fBmysql\-client\fR on architecture \fBi386\fR, architecture \fB
 \fBmysql\-client (>= 3\.6) {i386}\fR
 version and architecture conditions combined\.
 .
+.TP
+\fBlibmysqlclient18_5\.5\.35\-rel33\.0\-611\.squeeze_amd64\fR
+direct package reference\.
+.
+.TP
+\fB$Source (nginx)\fR
+all binary packages with \fBnginx\fR as source package\.
+.
+.TP
+\fB!Name (~ \.*\-dev), mail\-transport, $Version (>= 3\.5)\fR
+matches all packages that provide \fBmail\-transport\fR with name that has no suffix \fB\-dev\fR and with version greater or equal to \fB3\.5\fR\.
+.
 .P
-When specified on command line, condition may have to be quoted according to shell rules, so that it stays single argument:
+When specified on command line, query may have to be quoted according to shell rules, so that it stays single argument:
 .
 .P
 \fBaptly repo import percona stable \(cqmysql\-client (>= 3\.6)\(cq\fR
@@ -184,6 +292,14 @@ $ aptly mirror create wheezy\-main http://mirror\.yandex\.ru/debian/ wheezy main
 Options:
 .
 .TP
+\-\fBfilter\fR=
+filter packages in mirror
+.
+.TP
+\-\fBfilter\-with\-deps\fR=false
+when filtering, include dependencies of matching packages as well
+.
+.TP
 \-\fBignore\-signatures\fR=false
 disable verification of Release file signatures
 .
@@ -268,6 +384,10 @@ $ aptly mirror update wheezy\-main
 Options:
 .
 .TP
+\-\fBdownload\-limit\fR=0
+limit download speed (kbytes/sec)
+.
+.TP
 \-\fBignore\-checksums\fR=false
 ignore checksum mismatches while downloading package files and metadata
 .
@@ -279,6 +399,41 @@ disable verification of Release file signatures
 \-\fBkeyring\fR=
 gpg keyring to use when verifying Release file (could be specified multiple times)
 .
+.SH "RENAMES MIRROR"
+\fBaptly\fR \fBmirror\fR \fBrename\fR \fIold\-name\fR \fInew\-name\fR
+.
+.P
+Command changes name of the mirror\.Mirror name should be unique\.
+.
+.P
+Example:
+.
+.P
+$ aptly mirror rename wheezy\-min wheezy\-main
+.
+.SH "EDIT PROPERTIES OF MIRORR"
+\fBaptly\fR \fBmirror\fR \fBedit\fR \fIname\fR
+.
+.P
+Command edit allows to change settings of mirror: filters\.
+.
+.P
+Example:
+.
+.P
+$ aptly mirror edit \-filter=nginx \-filter\-with\-deps some\-mirror
+.
+.P
+Options:
+.
+.TP
+\-\fBfilter\fR=
+filter packages in mirror
+.
+.TP
+\-\fBfilter\-with\-deps\fR=false
+when filtering, include dependencies of matching packages as well
+.
 .SH "ADD PACKAGES TO LOCAL REPOSITORY"
 \fBaptly\fR \fBrepo\fR \fBadd\fR \fIname\fR
 .
@@ -299,10 +454,10 @@ Options:
 remove files that have been imported successfully into repository
 .
 .SH "COPY PACKAGES BETWEEN LOCAL REPOSITORIES"
-\fBaptly\fR \fBrepo\fR \fBcopy\fR \fIsrc\-name\fR \fIdst\-name\fR \fIpackage\-spec\fR \fB\|\.\|\.\|\.\fR
+\fBaptly\fR \fBrepo\fR \fBcopy\fR \fIsrc\-name\fR \fIdst\-name\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
 .
 .P
-Command copy copies packages matching \fIpackage\-spec\fR from local repo \fIsrc\-name\fR to local repo \fIdst\-name\fR\.
+Command copy copies packages matching \fIpackage\-query\fR from local repo \fIsrc\-name\fR to local repo \fIdst\-name\fR\.
 .
 .P
 Example:
@@ -395,10 +550,10 @@ default component when publishing
 default distribution when publishing
 .
 .SH "IMPORT PACKAGES FROM MIRROR TO LOCAL REPOSITORY"
-\fBaptly\fR \fBrepo\fR \fBimport\fR \fIsrc\-mirror\fR \fIdst\-repo\fR \fIpackage\-spec\fR \fB\|\.\|\.\|\.\fR
+\fBaptly\fR \fBrepo\fR \fBimport\fR \fIsrc\-mirror\fR \fIdst\-repo\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
 .
 .P
-Command import looks up packages matching \fIpackage\-spec\fR in mirror \fIsrc\-mirror\fR and copies them to local repo \fIdst\-repo\fR\.
+Command import looks up packages matching \fIpackage\-query\fR in mirror \fIsrc\-mirror\fR and copies them to local repo \fIdst\-repo\fR\.
 .
 .P
 Example:
@@ -437,10 +592,10 @@ Options:
 display list in machine\-readable format
 .
 .SH "MOVE PACKAGES BETWEEN LOCAL REPOSITORIES"
-\fBaptly\fR \fBrepo\fR \fBmove\fR \fIsrc\-name\fR \fIdst\-name\fR \fIpackage\-spec\fR \fB\|\.\|\.\|\.\fR
+\fBaptly\fR \fBrepo\fR \fBmove\fR \fIsrc\-name\fR \fIdst\-name\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
 .
 .P
-Command move moves packages matching \fIpackage\-spec\fR from local repo \fIsrc\-name\fR to local repo \fIdst\-name\fR\.
+Command move moves packages matching \fIpackage\-query\fR from local repo \fIsrc\-name\fR to local repo \fIdst\-name\fR\.
 .
 .P
 Example:
@@ -460,10 +615,10 @@ don\(cqt move, just show what would be moved
 follow dependencies when processing package\-spec
 .
 .SH "REMOVE PACKAGES FROM LOCAL REPOSITORY"
-\fBaptly\fR \fBrepo\fR \fBremove\fR \fIname\fR \fIpackage\-spec\fR \fB\|\.\|\.\|\.\fR
+\fBaptly\fR \fBrepo\fR \fBremove\fR \fIname\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
 .
 .P
-Commands removes packages matching \fIpackage\-spec\fR from local repository \fIname\fR\. If removed packages are not referenced by other repos or snapshots, they can be removed completely (including files) by running \(cqaptly db cleanup\(cq\.
+Commands removes packages matching \fIpackage\-query\fR from local repository \fIname\fR\. If removed packages are not referenced by other repos or snapshots, they can be removed completely (including files) by running \(cqaptly db cleanup\(cq\.
 .
 .P
 Example:
@@ -494,6 +649,18 @@ Options:
 \-\fBwith\-packages\fR=false
 show list of packages
 .
+.SH "RENAMES LOCAL REPOSITORY"
+\fBaptly\fR \fBrepo\fR \fBrename\fR \fIold\-name\fR \fInew\-name\fR
+.
+.P
+Command changes name of the local repo\. Local repo name should be unique\.
+.
+.P
+Example:
+.
+.P
+$ aptly repo rename wheezy\-min wheezy\-main
+.
 .SH "CREATES SNAPSHOT OF MIRROR (LOCAL REPOSITORY) CONTENTS"
 \fBaptly\fR \fBsnapshot\fR \fBcreate\fR \fIname\fR \fBfrom\fR \fBmirror\fR \fImirror\-name\fR \fB|\fR \fBfrom\fR \fBrepo\fR \fIrepo\-name\fR \fB|\fR \fBempty\fR
 .
@@ -531,6 +698,10 @@ Options:
 \-\fBraw\fR=false
 display list in machine\-readable format
 .
+.TP
+\-\fBsort\fR=name
+display list in \(cqname\(cq or creation \(cqtime\(cq order
+.
 .SH "SHOWS DETAILS ABOUT SNAPSHOT"
 \fBaptly\fR \fBsnapshot\fR \fBshow\fR \fIname\fR
 .
@@ -577,10 +748,10 @@ $ aptly snapshot verify wheezy\-main wheezy\-contrib wheezy\-non\-free
 .IP "" 0
 .
 .SH "PULL PACKAGES FROM ANOTHER SNAPSHOT"
-\fBaptly\fR \fBsnapshot\fR \fBpull\fR \fIname\fR \fIsource\fR \fIdestination\fR \fIpackage\-name\fR \fB\|\.\|\.\|\.\fR
+\fBaptly\fR \fBsnapshot\fR \fBpull\fR \fIname\fR \fIsource\fR \fIdestination\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
 .
 .P
-Command pull pulls new packages along with its\(cq dependencies to snapshot \fIname\fR from snapshot \fIsource\fR\. Pull can upgrade package version in \fIname\fR with versions from \fIsource\fR following dependencies\. New snapshot \fIdestination\fR is created as a result of this process\. Packages could be specified simply as \(cqpackage\-name\(cq or as dependency \(cqpackage\-name (>= version)\(cq\.
+Command pull pulls new packages along with its\(cq dependencies to snapshot \fIname\fR from snapshot \fIsource\fR\. Pull can upgrade package version in \fIname\fR with versions from \fIsource\fR following dependencies\. New snapshot \fIdestination\fR is created as a result of this process\. Packages could be specified simply as \(cqpackage\-name\(cq or as package queries\.
 .
 .P
 Example:
@@ -599,6 +770,10 @@ $ aptly snapshot pull wheezy\-main wheezy\-backports wheezy\-new\-xorg xorg\-ser
 Options:
 .
 .TP
+\-\fBall\-matches\fR=false
+pull all the packages that satisfy the dependency version requirements
+.
+.TP
 \-\fBdry\-run\fR=false
 don\(cqt create destination snapshot, just show what would be pulled
 .
@@ -662,6 +837,10 @@ Options:
 \-\fBlatest\fR=false
 use only the latest version of each package
 .
+.TP
+\-\fBno\-remove\fR=false
+don\(cqt remove duplicate arch/name packages
+.
 .SH "DELETE SNAPSHOT"
 \fBaptly\fR \fBsnapshot\fR \fBdrop\fR \fIname\fR
 .
@@ -688,11 +867,23 @@ Options:
 \-\fBforce\fR=false
 remove snapshot even if it was used as source for other snapshots
 .
-.SH "REMOVE PUBLISHED REPOSITORY"
-\fBaptly\fR \fBpublish\fR \fBdrop\fR \fIdistribution\fR [\fIprefix\fR]
+.SH "RENAMES SNAPSHOT"
+\fBaptly\fR \fBsnapshot\fR \fBrename\fR \fIold\-name\fR \fInew\-name\fR
 .
 .P
-Command removes whatever has been published under specified \fIprefix\fR and \fIdistribution\fR name\.
+Command changes name of the snapshot\. Snapshot name should be unique\.
+.
+.P
+Example:
+.
+.P
+$ aptly snapshot rename wheezy\-min wheezy\-main
+.
+.SH "REMOVE PUBLISHED REPOSITORY"
+\fBaptly\fR \fBpublish\fR \fBdrop\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR]
+.
+.P
+Command removes whatever has been published under specified \fIprefix\fR, publishing \fIendpoint\fR and \fIdistribution\fR name\.
 .
 .P
 Example:
@@ -734,12 +925,25 @@ Options:
 display list in machine\-readable format
 .
 .SH "PUBLISH LOCAL REPOSITORY"
-\fBaptly\fR \fBpublish\fR \fBrepo\fR \fIname\fR [\fIprefix\fR]
+\fBaptly\fR \fBpublish\fR \fBrepo\fR \fIname\fR [[\fIendpoint\fR:]\fIprefix\fR]
 .
 .P
 Command publishes current state of local repository ready to be consumed by apt tools\. Published repostiories appear under rootDir/public directory\. Valid GPG key is required for publishing\.
 .
 .P
+Multiple component repository could be published by specifying several components split by commas via \-component flag and multiple local repositories as the arguments:
+.
+.IP "" 4
+.
+.nf
+
+aptly publish repo \-component=main,contrib repo\-main repo\-contrib
+.
+.fi
+.
+.IP "" 0
+.
+.P
 It is not recommended to publish local repositories directly unless the repository is for testing purposes and changes happen frequently\. For production usage please take snapshot of repository and publish it using publish snapshot command\.
 .
 .P
@@ -760,13 +964,17 @@ Options:
 .
 .TP
 \-\fBcomponent\fR=
-component name to publish
+component name to publish (for multi\-component publishing, separate components with commas)
 .
 .TP
 \-\fBdistribution\fR=
 distribution name to publish
 .
 .TP
+\-\fBforce\-overwrite\fR=false
+overwrite files in package pool in case of mismatch
+.
+.TP
 \-\fBgpg\-key\fR=
 GPG key ID to use when signing the release
 .
@@ -791,12 +999,25 @@ GPG secret keyring to use (instead of default)
 don\(cqt sign Release files with GPG
 .
 .SH "PUBLISH SNAPSHOT"
-\fBaptly\fR \fBpublish\fR \fBsnapshot\fR \fIname\fR [\fIprefix\fR]
+\fBaptly\fR \fBpublish\fR \fBsnapshot\fR \fIname\fR [[\fIendpoint\fR:]\fIprefix\fR]
 .
 .P
 Command publishes snapshot as Debian repository ready to be consumed by apt tools\. Published repostiories appear under rootDir/public directory\. Valid GPG key is required for publishing\.
 .
 .P
+Multiple component repository could be published by specifying several components split by commas via \-component flag and multiple snapshots as the arguments:
+.
+.IP "" 4
+.
+.nf
+
+aptly publish snapshot \-component=main,contrib snap\-main snap\-contrib
+.
+.fi
+.
+.IP "" 0
+.
+.P
 Example:
 .
 .IP "" 4
@@ -814,13 +1035,17 @@ Options:
 .
 .TP
 \-\fBcomponent\fR=
-component name to publish
+component name to publish (for multi\-component publishing, separate components with commas)
 .
 .TP
 \-\fBdistribution\fR=
 distribution name to publish
 .
 .TP
+\-\fBforce\-overwrite\fR=false
+overwrite files in package pool in case of mismatch
+.
+.TP
 \-\fBgpg\-key\fR=
 GPG key ID to use when signing the release
 .
@@ -845,12 +1070,25 @@ GPG secret keyring to use (instead of default)
 don\(cqt sign Release files with GPG
 .
 .SH "UPDATE PUBLISHED REPOSITORY BY SWITCHING TO NEW SNAPSHOT"
-\fBaptly\fR \fBpublish\fR \fBswitch\fR \fIdistribution\fR [\fIprefix\fR] \fInew\-snapshot\fR
+\fBaptly\fR \fBpublish\fR \fBswitch\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR] \fInew\-snapshot\fR
 .
 .P
 Command switches in\-place published repository with new snapshot contents\. All publishing parameters are preserved (architecture list, distribution, component)\.
 .
 .P
+For multiple component repositories, flag \-component should be given with list of components to update\. Corresponding snapshots should be given in the same order, e\.g\.:
+.
+.IP "" 4
+.
+.nf
+
+aptly publish update \-component=main,contrib wheezy wh\-main wh\-contrib
+.
+.fi
+.
+.IP "" 0
+.
+.P
 Example:
 .
 .IP "" 4
@@ -867,6 +1105,14 @@ $ aptly publish update wheezy ppa wheezy\-7\.5
 Options:
 .
 .TP
+\-\fBcomponent\fR=
+component names to update (for multi\-component publishing, separate components with commas)
+.
+.TP
+\-\fBforce\-overwrite\fR=false
+overwrite files in package pool in case of mismatch
+.
+.TP
 \-\fBgpg\-key\fR=
 GPG key ID to use when signing the release
 .
@@ -883,12 +1129,15 @@ GPG secret keyring to use (instead of default)
 don\(cqt sign Release files with GPG
 .
 .SH "UPDATE PUBLISHED LOCAL REPOSITORY"
-\fBaptly\fR \fBpublish\fR \fBupdate\fR \fIdistribution\fR [\fIprefix\fR]
+\fBaptly\fR \fBpublish\fR \fBupdate\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR]
 .
 .P
 Command re\-publishes (updates) published local repository\. \fIdistribution\fR and \fIprefix\fR should be occupied with local repository published using command aptly publish repo\. Update happens in\-place with minimum possible downtime for published repository\.
 .
 .P
+For multiple component published repositories, all local repositories are updated\.
+.
+.P
 Example:
 .
 .IP "" 4
@@ -905,6 +1154,10 @@ $ aptly publish update wheezy ppa
 Options:
 .
 .TP
+\-\fBforce\-overwrite\fR=false
+overwrite files in package pool in case of mismatch
+.
+.TP
 \-\fBgpg\-key\fR=
 GPG key ID to use when signing the release
 .
@@ -979,7 +1232,19 @@ $ aptly graph
 If environment variable \fBHTTP_PROXY\fR is set \fBaptly\fR would use its value to proxy all HTTP requests\.
 .
 .SH "RETURN VALUES"
-\fBaptly\fR exists with 0 on success and with 1 on failure\.
+\fBaptly\fR exists with:
+.
+.TP
+0
+success
+.
+.TP
+1
+general failure
+.
+.TP
+2
+command parse failure
 .
 .SH "AUTHORS"
 Andrey Smirnov (me@smira\.ru)

man/aptly.1.ronn.tmpl

@@ -28,6 +28,7 @@ Configuration file is stored in JSON format (default values shown below):
     {
       "rootDir": "$HOME/.aptly",
       "downloadConcurrency": 4,
+      "downloadSpeedLimit": 0,
       "architectures": [],
       "dependencyFollowSuggests": false,
       "dependencyFollowRecommends": false
@@ -37,7 +38,16 @@ Configuration file is stored in JSON format (default values shown below):
       "gpgDisableVerify": false,
       "downloadSourcePackages": false,
       "ppaDistributorID": "ubuntu",
-      "ppaCodename": ""
+      "ppaCodename": "",
+      "S3PublishEndpoints": {
+        "test": {
+          "region": "us-east-1",
+          "bucket": "repo",
+          "awsAccessKeyID": ""
+          "awsSecretAccessKey": "",
+          "prefix": "",
+          "acl": "public-read"
+        }
     }
 
 Options:
@@ -49,6 +59,9 @@ Options:
   * `downloadConcurrency`:
     is a number of parallel download threads to use when downloading packages
 
+  * `downloadSpeedLimit`:
+    limit in kbytes/sec on download speed while mirroring remote repositieis
+
   * `architectures`:
     is a list of architectures to process; if left empty defaults to all available architectures; could be
     overridden with option `-architectures`
@@ -81,10 +94,43 @@ Options:
     specifies paramaters for short PPA url expansion, if left blank they default
     to output of `lsb_release` command
 
-## PACKAGE SPEC
+  * `S3PublisEndpoints`:
+    configuration of Amazon S3 publishing endpoints (see below)
 
-Some commands accept package specs to identify list of packages to process.
-Package spec is a list of following search conditions:
+## S3 PUBLISHING ENDPOINTS
+
+aptly could be configured to publish repository directly to Amazon S3. First, publishing
+endpoints should be described in aptly configuration file. Each endpoint has name
+and associated settings:
+
+   * `region`:
+     Amazon region for S3 bucket (e.g. `us-east-1`)
+   * `bucket`:
+     bucket name
+   * `prefix`:
+     (optional) do publishing under specified prefix in the bucket, defaults to
+     no prefix (bucket root)
+   * `acl`:
+     (optional) assign ACL to published files (one of the canned ACLs in Amazon
+     terminology). Useful values: `private` (default) or `public-read` (public
+     repository). Public repositories could be consumed by `apt` using
+     HTTP endpoint (Amazon bucket should be configured for "website hosting"),
+     for private repositories special apt S3 transport is required.
+   * `awsAccessKeyID`, `awsSecretAccessKey`:
+     (optional) Amazon credentials to access S3 bucket. If not supplied,
+     environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+     are used.
+
+In order to publish to S3, specify endpoint as `s3:endpoint-name:` before
+publishing prefix on the command line, e.g.:
+
+  `aptly publish snapshot wheezy-main s3:test:`
+
+## PACKAGE QUERY
+
+Some commands accept package queries to identify list of packages to process.
+Package query syntax almost matches `reprepro` query language. Query consists of
+the following simple terms:
 
   * direct package reference:
     reference to exaclty one package. Format is identical to the way aptly lists packages in
@@ -92,12 +138,49 @@ Package spec is a list of following search conditions:
     e.g.: `libmysqlclient18_5.5.35-rel33.0-611.squeeze_amd64`
 
   * dependency condition:
-    syntax follows Debian dependency specification: package_name followed by optional version specification and architecture limit.
+    syntax follows Debian dependency specification: package_name followed by optional version specification
+    and architecture limit, e.g: `mysql-client (>= 3.6)`.
+
+  * query against package fields:
+    syntax is the same as for dependency conditions, but instead of package name field name is used, e.g:
+    `Priority (optional)`.
+
+Supported fields:
+
+  * all field names from Debian package control files are supported except for `Filename`, `MD5sum`,
+    `SHA1`, `SHA256`, `Size`, `Files`, `Checksums-SHA1`, `Checksums-SHA256`.
+  * `$Source` is a name of source package (for binary packages)
+  * `$SourceVersion` is a version of source package
+  * `$Architecture` is `Architecture` for binary packages and `source` for source packages,
+     when matching with equal (`=`) operator, package with `any` architecture matches all architectures
+     but `source`.
+  * `$Version` has the same value as `Version`, but comparison operators use Debian
+     version precedence rules
+  * `$PackageType` is `deb` for binary packages and `source` for source packages
+
+Operators:
+
+  * `=`:
+    strict match, default operator is no operator is given
+  * `>=`, `<=`, `=`, `>>` (strictly greater), `<<` (strictly less):
+    lexicographical comparison for all fields and special rules when comparing package versions
+  * `%`:
+    pattern matching, like shell patterns, supported special symbols are: `[^]?*`, e.g.:
+    `$Version (% 3.5-*)`
+  * `~`:
+    regular expression matching, e.g.:
+    `Name (~ .*-dev)`
+
+Simple terms could be combined into more complex queries using operators `,` (and), `|` (or) and
+`!` (not), parentheses `()` are used to change operator precedence. Match value could be
+enclosed in single (`'`) or double (`"`) quotes if required to resolve ambiguity, quotes
+inside quoted string should escaped with slash (`\`).
 
 Examples:
 
   * `mysql-client`:
-     matches package mysql-client of any version and architecture (including source)
+     matches package mysql-client of any version and architecture (including source), also
+     matches packages that `Provide:` `mysql-client`.
 
   * `mysql-client (>= 3.6)`:
      matches package mysql-client with version greater or equal to 3.6. Valid operators for
@@ -107,9 +190,19 @@ Examples:
      matches package `mysql-client` on architecture `i386`, architecture `all` matches all architectures but source.
 
   * `mysql-client (>= 3.6) {i386}`:
-      version and architecture conditions combined.
+    version and architecture conditions combined.
 
-When specified on command line, condition may have to be quoted according to shell rules, so that it stays single argument:
+  * `libmysqlclient18_5.5.35-rel33.0-611.squeeze_amd64`:
+    direct package reference.
+
+  * `$Source (nginx)`:
+    all binary packages with `nginx` as source package.
+
+  * `!Name (~ .*-dev), mail-transport, $Version (>= 3.5)`:
+    matches all packages that provide `mail-transport` with name that has no suffix `-dev` and
+    with version greater or equal to `3.5`.
+
+When specified on command line, query may have to be quoted according to shell rules, so that it stays single argument:
 
   `aptly repo import percona stable 'mysql-client (>= 3.6)'`
 
@@ -139,7 +232,16 @@ to proxy all HTTP requests.
 
 ## RETURN VALUES
 
-`aptly` exists with 0 on success and with 1 on failure.
+`aptly` exists with:
+
+ * 0:
+   success
+
+ * 1:
+   general failure
+
+ * 2:
+   command parse failure
 
 ## AUTHORS
 

query/lex.go

@@ -0,0 +1,247 @@
+package query
+
+import (
+	"fmt"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// itemType identifies the type of lex items.
+type itemType int
+
+const eof = -1
+
+const (
+	itemNull  itemType = iota
+	itemError          // error occurred;
+	// value is text of error
+	itemEOF
+	itemLeftParen  // (
+	itemRightParen // )
+	itemOr         // |
+	itemAnd        // ,
+	itemNot        // !
+	itemLt         // <<
+	itemLtEq       // <=, <
+	itemGt         // >>
+	itemGtEq       // >=, >
+	itemEq         // =
+	itemPatMatch   // %
+	itemRegexp     // ~
+	itemLeftCurly  // {
+	itemRightCurly // }
+	itemString
+)
+
+// item represents a token returned from the scanner.
+type item struct {
+	typ itemType // Type, such as itemNumber.
+	val string   // Value, such as "23.2".
+}
+
+func (i item) String() string {
+	if i.typ == itemString {
+		return fmt.Sprintf("%#v", i.val)
+	}
+	if i.typ == itemEOF {
+		return "<EOL>"
+	}
+	if i.typ == itemError {
+		return fmt.Sprintf("error: %s", i.val)
+	}
+	if i.typ == itemNull {
+		return "<NULL>"
+	}
+	return i.val
+}
+
+// stateFn represents the state of the scanner
+// as a function that returns the next state.
+type stateFn func(*lexer) stateFn
+
+// lexer holds the state of the scanner.
+type lexer struct {
+	name  string    // used only for error reports.
+	input string    // the string being scanned.
+	start int       // start position of this item.
+	pos   int       // current position in the input.
+	width int       // width of last rune read from input.
+	items chan item // channel of scanned items.
+	last  item
+}
+
+func lex(name, input string) (*lexer, chan item) {
+	l := &lexer{
+		name:  name,
+		input: input,
+		items: make(chan item),
+	}
+	go l.run() // Concurrently run state machine.
+	return l, l.items
+}
+
+// emit passes an item back to the client.
+func (l *lexer) emit(t itemType) {
+	l.items <- item{t, l.input[l.start:l.pos]}
+	l.start = l.pos
+}
+
+// run lexes the input by executing state functions until
+// the state is nil.
+func (l *lexer) run() {
+	for state := lexMain; state != nil; {
+		state = state(l)
+	}
+	close(l.items) // No more tokens will be delivered.
+}
+
+// next returns the next rune in the input.
+func (l *lexer) next() (r rune) {
+	if l.pos >= len(l.input) {
+		l.width = 0
+		return eof
+	}
+	r, l.width =
+		utf8.DecodeRuneInString(l.input[l.pos:])
+	l.pos += l.width
+	return r
+}
+
+// ignore skips over the pending input before this point.
+func (l *lexer) ignore() {
+	l.start = l.pos
+}
+
+// backup steps back one rune.
+// Can be called only once per call of next.
+func (l *lexer) backup() {
+	l.pos -= l.width
+}
+
+// peek returns but does not consume
+// the next rune in the input.
+func (l *lexer) peek() rune {
+	r := l.next()
+	l.backup()
+	return r
+}
+
+func (l *lexer) Current() item {
+	if l.last.typ == 0 {
+		l.last = <-l.items
+	}
+
+	return l.last
+}
+
+func (l *lexer) Consume() {
+	l.last = <-l.items
+}
+
+// error returns an error token and terminates the scan
+// by passing back a nil pointer that will be the next
+// state, terminating l.run.
+func (l *lexer) errorf(format string, args ...interface{}) stateFn {
+	l.items <- item{
+		itemError,
+		fmt.Sprintf(format, args...),
+	}
+	return nil
+}
+
+func lexMain(l *lexer) stateFn {
+	switch r := l.next(); {
+	case r == eof:
+		l.emit(itemEOF)
+		return nil
+	case unicode.IsSpace(r):
+		l.ignore()
+	case r == '(':
+		l.emit(itemLeftParen)
+	case r == ')':
+		l.emit(itemRightParen)
+	case r == '{':
+		l.emit(itemLeftCurly)
+	case r == '}':
+		l.emit(itemRightCurly)
+	case r == '|':
+		l.emit(itemOr)
+	case r == ',':
+		l.emit(itemAnd)
+	case r == '!':
+		l.emit(itemNot)
+	case r == '<':
+		r2 := l.next()
+		if r2 == '<' {
+			l.emit(itemLt)
+		} else if r2 == '=' {
+			l.emit(itemLtEq)
+		} else {
+			l.backup()
+			l.emit(itemLtEq)
+		}
+	case r == '>':
+		r2 := l.next()
+		if r2 == '>' {
+			l.emit(itemGt)
+		} else if r2 == '=' {
+			l.emit(itemGtEq)
+		} else {
+			l.backup()
+			l.emit(itemGtEq)
+		}
+	case r == '=':
+		l.emit(itemEq)
+	case r == '%':
+		l.emit(itemPatMatch)
+	case r == '~':
+		l.emit(itemRegexp)
+	default:
+		l.backup()
+		return lexString
+	}
+
+	return lexMain
+}
+
+func lexString(l *lexer) stateFn {
+	r := l.next()
+	// quoted string
+	if r == '"' || r == '\'' {
+		quote := r
+		result := ""
+		l.ignore()
+		for {
+			r = l.next()
+			if r == quote {
+				l.ignore()
+				l.items <- item{itemString, result}
+				return lexMain
+			}
+			if r == '\\' {
+				r = l.next()
+			}
+			if r == eof {
+				return l.errorf("unexpected eof in quoted string")
+			}
+			result = result + string(r)
+		}
+	} else {
+		// unquoted string
+		for {
+			if unicode.IsSpace(r) || strings.IndexRune("()|,!{}", r) > 0 {
+				l.backup()
+				l.emit(itemString)
+				return lexMain
+			}
+
+			if r == eof {
+				l.emit(itemString)
+				l.emit(itemEOF)
+				return nil
+			}
+			r = l.next()
+		}
+	}
+}

query/lex_test.go

@@ -0,0 +1,57 @@
+package query
+
+import (
+	"fmt"
+	. "launchpad.net/gocheck"
+)
+
+type LexerSuite struct {
+}
+
+var _ = Suite(&LexerSuite{})
+
+func (s *LexerSuite) TestLexing(c *C) {
+	_, ch := lex("query", "package (<< 1.3), $Source | !\"app\", 'd\"\\a\\'ta' {i386}")
+
+	c.Check(<-ch, Equals, item{typ: itemString, val: "package"})
+	c.Check(<-ch, Equals, item{typ: itemLeftParen, val: "("})
+	c.Check(<-ch, Equals, item{typ: itemLt, val: "<<"})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "1.3"})
+	c.Check(<-ch, Equals, item{typ: itemRightParen, val: ")"})
+	c.Check(<-ch, Equals, item{typ: itemAnd, val: ","})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "$Source"})
+	c.Check(<-ch, Equals, item{typ: itemOr, val: "|"})
+	c.Check(<-ch, Equals, item{typ: itemNot, val: "!"})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "app"})
+	c.Check(<-ch, Equals, item{typ: itemAnd, val: ","})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "d\"a'ta"})
+	c.Check(<-ch, Equals, item{typ: itemLeftCurly, val: "{"})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "i386"})
+	c.Check(<-ch, Equals, item{typ: itemRightCurly, val: "}"})
+	c.Check(<-ch, Equals, item{typ: itemEOF, val: ""})
+}
+
+func (s *LexerSuite) TestConsume(c *C) {
+	l, _ := lex("query", "package (<< 1.3)")
+
+	c.Check(l.Current(), Equals, item{typ: itemString, val: "package"})
+	c.Check(l.Current(), Equals, item{typ: itemString, val: "package"})
+	l.Consume()
+	c.Check(l.Current(), Equals, item{typ: itemLeftParen, val: "("})
+	l.Consume()
+	c.Check(l.Current(), Equals, item{typ: itemLt, val: "<<"})
+}
+
+func (s *LexerSuite) TestString(c *C) {
+	l, _ := lex("query", "package (<< 1.3)")
+
+	c.Check(fmt.Sprintf("%s", l.Current()), Equals, "\"package\"")
+	l.Consume()
+	c.Check(fmt.Sprintf("%s", l.Current()), Equals, "(")
+}
+
+func (s *LexerSuite) TestError(c *C) {
+	l, _ := lex("query", "'package")
+
+	c.Check(fmt.Sprintf("%s", l.Current()), Equals, "error: unexpected eof in quoted string")
+}

query/query.go

@@ -0,0 +1,29 @@
+// Package query implements query language for
+package query
+
+import (
+	"github.com/smira/aptly/deb"
+)
+
+/*
+
+  Query language resembling Debian dependencies and reprepro
+  queries: http://mirrorer.alioth.debian.org/reprepro.1.html
+
+  Query := A | A '|' Query
+  A := B | B ',' A
+  B := C | '!' B
+  C := '(' Query ')' | D
+  D := <field> <condition> <arch_condition> | <pkg>_<version>_<arch>
+  field := <package-name> | <field> | $special_field
+  condition := '(' <operator> value ')' |
+  arch_condition := '{' arch '}' |
+  operator := | << | < | <= | > | >> | >= | = | % | ~
+*/
+
+// Parse parses input package query into PackageQuery tree ready for evaluation
+func Parse(query string) (result deb.PackageQuery, err error) {
+	l, _ := lex("", query)
+	result, err = parse(l)
+	return
+}

query/query_test.go

@@ -0,0 +1,11 @@
+package query
+
+import (
+	. "launchpad.net/gocheck"
+	"testing"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

query/syntax.go

@@ -0,0 +1,225 @@
+package query
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"regexp"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+type parser struct {
+	name  string // used only for error reports.
+	input *lexer // the input lexer
+	err   error  // error stored while parsing
+}
+
+func parse(input *lexer) (deb.PackageQuery, error) {
+	p := &parser{
+		name:  input.name,
+		input: input,
+	}
+	query := p.parse()
+	if p.err != nil {
+		return nil, p.err
+	}
+	return query, nil
+}
+
+// Entry into parser
+func (p *parser) parse() deb.PackageQuery {
+	defer func() {
+		if r := recover(); r != nil {
+			p.err = fmt.Errorf("parsing failed: %s", r)
+		}
+	}()
+
+	q := p.Query()
+	if p.input.Current().typ != itemEOF {
+		panic(fmt.Sprintf("unexpected token %s: expecting end of query", p.input.Current()))
+	}
+	return q
+}
+
+// Query := A | A '|' Query
+func (p *parser) Query() deb.PackageQuery {
+	q := p.A()
+	if p.input.Current().typ == itemOr {
+		p.input.Consume()
+		return &deb.OrQuery{L: q, R: p.Query()}
+	}
+	return q
+}
+
+// A := B | B ',' A
+func (p *parser) A() deb.PackageQuery {
+	q := p.B()
+	if p.input.Current().typ == itemAnd {
+		p.input.Consume()
+		return &deb.AndQuery{L: q, R: p.A()}
+	}
+	return q
+}
+
+// B := C | '!' B
+func (p *parser) B() deb.PackageQuery {
+	if p.input.Current().typ == itemNot {
+		p.input.Consume()
+		return &deb.NotQuery{Q: p.B()}
+	}
+	return p.C()
+}
+
+// C := '(' Query ')' | D
+func (p *parser) C() deb.PackageQuery {
+	if p.input.Current().typ == itemLeftParen {
+		p.input.Consume()
+		q := p.Query()
+		if p.input.Current().typ != itemRightParen {
+			panic(fmt.Sprintf("unexpected token %s: expecting ')'", p.input.Current()))
+		}
+		p.input.Consume()
+		return q
+	}
+	return p.D()
+}
+
+func operatorToRelation(operator itemType) int {
+	switch operator {
+	case 0:
+		return deb.VersionDontCare
+	case itemLt:
+		return deb.VersionLess
+	case itemLtEq:
+		return deb.VersionLessOrEqual
+	case itemGt:
+		return deb.VersionGreater
+	case itemGtEq:
+		return deb.VersionGreaterOrEqual
+	case itemEq:
+		return deb.VersionEqual
+	case itemPatMatch:
+		return deb.VersionPatternMatch
+	case itemRegexp:
+		return deb.VersionRegexp
+	}
+	panic("unable to map token to relation")
+}
+
+// isPackageRef returns ok true if field has format pkg_version_arch
+func parsePackageRef(query string) (pkg, version, arch string, ok bool) {
+	i := strings.Index(query, "_")
+	if i != -1 {
+		pkg, query = query[:i], query[i+1:]
+		j := strings.LastIndex(query, "_")
+		if j != -1 {
+			version, arch = query[:j], query[j+1:]
+			ok = true
+		}
+	}
+	return
+}
+
+// D := <field> <condition> <arch_condition> | <package>_<version>_<arch>
+// field := <package-name> | <field> | $special_field
+func (p *parser) D() deb.PackageQuery {
+	if p.input.Current().typ != itemString {
+		panic(fmt.Sprintf("unexpected token %s: expecting field or package name", p.input.Current()))
+	}
+
+	field := p.input.Current().val
+	p.input.Consume()
+
+	operator, value := p.Condition()
+
+	r, _ := utf8.DecodeRuneInString(field)
+	if strings.HasPrefix(field, "$") || unicode.IsUpper(r) {
+		// special field or regular field
+		q := &deb.FieldQuery{Field: field, Relation: operatorToRelation(operator), Value: value}
+		if q.Relation == deb.VersionRegexp {
+			var err error
+			q.Regexp, err = regexp.Compile(q.Value)
+			if err != nil {
+				panic(fmt.Sprintf("regexp compile failed: %s", err))
+			}
+		}
+		return q
+	} else if operator == 0 && value == "" {
+		if pkg, version, arch, ok := parsePackageRef(field); ok {
+			// query for specific package
+			return &deb.PkgQuery{Pkg: pkg, Version: version, Arch: arch}
+		}
+	}
+
+	// regular dependency-like query
+	q := &deb.DependencyQuery{Dep: deb.Dependency{
+		Pkg:          field,
+		Relation:     operatorToRelation(operator),
+		Version:      value,
+		Architecture: p.ArchCondition()}}
+	if q.Dep.Relation == deb.VersionRegexp {
+		var err error
+		q.Dep.Regexp, err = regexp.Compile(q.Dep.Version)
+		if err != nil {
+			panic(fmt.Sprintf("regexp compile failed: %s", err))
+		}
+	}
+	return q
+}
+
+// condition := '(' <operator> value ')' |
+// operator := | << | < | <= | > | >> | >= | = | % | ~
+func (p *parser) Condition() (operator itemType, value string) {
+	if p.input.Current().typ != itemLeftParen {
+		return
+	}
+	p.input.Consume()
+
+	if p.input.Current().typ == itemLt ||
+		p.input.Current().typ == itemGt ||
+		p.input.Current().typ == itemLtEq ||
+		p.input.Current().typ == itemGtEq ||
+		p.input.Current().typ == itemEq ||
+		p.input.Current().typ == itemPatMatch ||
+		p.input.Current().typ == itemRegexp {
+		operator = p.input.Current().typ
+		p.input.Consume()
+	} else {
+		operator = itemEq
+	}
+
+	if p.input.Current().typ != itemString {
+		panic(fmt.Sprintf("unexpected token %s: expecting value", p.input.Current()))
+	}
+	value = p.input.Current().val
+	p.input.Consume()
+
+	if p.input.Current().typ != itemRightParen {
+		panic(fmt.Sprintf("unexpected token %s: expecting ')'", p.input.Current()))
+	}
+	p.input.Consume()
+
+	return
+}
+
+// arch_condition := '{' arch '}' |
+func (p *parser) ArchCondition() (arch string) {
+	if p.input.Current().typ != itemLeftCurly {
+		return
+	}
+	p.input.Consume()
+
+	if p.input.Current().typ != itemString {
+		panic(fmt.Sprintf("unexpected token %s: expecting architecture", p.input.Current()))
+	}
+	arch = p.input.Current().val
+	p.input.Consume()
+
+	if p.input.Current().typ != itemRightCurly {
+		panic(fmt.Sprintf("unexpected token %s: expecting '}'", p.input.Current()))
+	}
+	p.input.Consume()
+
+	return
+}

query/syntax_test.go

@@ -0,0 +1,98 @@
+package query
+
+import (
+	"github.com/smira/aptly/deb"
+	. "launchpad.net/gocheck"
+	"regexp"
+)
+
+type SyntaxSuite struct {
+}
+
+var _ = Suite(&SyntaxSuite{})
+
+func (s *SyntaxSuite) TestParsing(c *C) {
+	l, _ := lex("query", "package (<< 1.3~dev), $Source")
+	q, err := parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q.(*deb.AndQuery).L, DeepEquals, &deb.DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionLess, Version: "1.3~dev"}})
+	c.Check(q.(*deb.AndQuery).R, DeepEquals, &deb.FieldQuery{Field: "$Source"})
+
+	l, _ = lex("query", "package (1.3), Name (lala) | !$Source")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q.(*deb.OrQuery).L.(*deb.AndQuery).L, DeepEquals, &deb.DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionEqual, Version: "1.3"}})
+	c.Check(q.(*deb.OrQuery).L.(*deb.AndQuery).R, DeepEquals, &deb.FieldQuery{Field: "Name", Relation: deb.VersionEqual, Value: "lala"})
+	c.Check(q.(*deb.OrQuery).R.(*deb.NotQuery).Q, DeepEquals, &deb.FieldQuery{Field: "$Source"})
+
+	l, _ = lex("query", "package, ((!(Name | $Source (~ a.*))))")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q.(*deb.AndQuery).L, DeepEquals, &deb.DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionDontCare}})
+	c.Check(q.(*deb.AndQuery).R.(*deb.NotQuery).Q.(*deb.OrQuery).L, DeepEquals, &deb.FieldQuery{Field: "Name", Relation: deb.VersionDontCare})
+	c.Check(q.(*deb.AndQuery).R.(*deb.NotQuery).Q.(*deb.OrQuery).R, DeepEquals, &deb.FieldQuery{Field: "$Source", Relation: deb.VersionRegexp, Value: "a.*",
+		Regexp: regexp.MustCompile("a.*")})
+
+	l, _ = lex("query", "package (> 5.3.7)")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q, DeepEquals, &deb.DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionGreaterOrEqual, Version: "5.3.7"}})
+
+	l, _ = lex("query", "package (~ 5\\.3.*~dev)")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q, DeepEquals, &deb.DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionRegexp, Version: "5\\.3.*~dev",
+		Regexp: regexp.MustCompile("5\\.3.*~dev")}})
+
+	l, _ = lex("query", "alien-data_1.3.4~dev_i386")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q, DeepEquals, &deb.PkgQuery{Pkg: "alien-data", Version: "1.3.4~dev", Arch: "i386"})
+
+	l, _ = lex("query", "package (> 5.3.7) {amd64}")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q, DeepEquals, &deb.DependencyQuery{
+		Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionGreaterOrEqual, Version: "5.3.7", Architecture: "amd64"}})
+}
+
+func (s *SyntaxSuite) TestParsingErrors(c *C) {
+	l, _ := lex("query", "package (> 5.3.7), ")
+	_, err := parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: unexpected token <EOL>: expecting field or package name")
+
+	l, _ = lex("query", "package>5.3.7)")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: unexpected token \\): expecting end of query")
+
+	l, _ = lex("query", "package | !|")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: unexpected token |: expecting field or package name")
+
+	l, _ = lex("query", "((package )")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: unexpected token <EOL>: expecting '\\)'")
+
+	l, _ = lex("query", "!package )")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: unexpected token \\): expecting end of query")
+
+	l, _ = lex("query", "'package )")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: unexpected token error: unexpected eof in quoted string: expecting field or package name")
+
+	l, _ = lex("query", "package (~ 1.2[34)")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: regexp compile failed: error parsing regexp: missing closing \\]: `\\[34`")
+
+	l, _ = lex("query", "$Name (~ 1.2[34)")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: regexp compile failed: error parsing regexp: missing closing \\]: `\\[34`")
+}

s3/public.go

@@ -0,0 +1,224 @@
+package s3
+
+import (
+	"fmt"
+	"github.com/mitchellh/goamz/aws"
+	"github.com/mitchellh/goamz/s3"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/files"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// PublishedStorage abstract file system with published files (actually hosted on S3)
+type PublishedStorage struct {
+	s3     *s3.S3
+	bucket *s3.Bucket
+	acl    s3.ACL
+	prefix string
+}
+
+// Check interface
+var (
+	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+)
+
+// NewPublishedStorageRaw creates published storage from raw aws credentials
+func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL, prefix string) (*PublishedStorage, error) {
+	if defaultACL == "" {
+		defaultACL = "private"
+	}
+
+	result := &PublishedStorage{s3: s3.New(auth, region), acl: s3.ACL(defaultACL), prefix: prefix}
+	result.bucket = result.s3.Bucket(bucket)
+
+	return result, nil
+}
+
+// NewPublishedStorage creates new instance of PublishedStorage with specified S3 access
+// keys, region and bucket name
+func NewPublishedStorage(accessKey, secretKey, region, bucket, defaultACL, prefix string) (*PublishedStorage, error) {
+	auth, err := aws.GetAuth(accessKey, secretKey)
+	if err != nil {
+		return nil, err
+	}
+
+	awsRegion, ok := aws.Regions[region]
+	if !ok {
+		return nil, fmt.Errorf("unknown region: %#v", region)
+	}
+
+	return NewPublishedStorageRaw(auth, awsRegion, bucket, defaultACL, prefix)
+}
+
+// String
+func (storage *PublishedStorage) String() string {
+	return fmt.Sprintf("S3: %s:%s/%s", storage.s3.Region.Name, storage.bucket.Name, storage.prefix)
+}
+
+// MkDir creates directory recursively under public path
+func (storage *PublishedStorage) MkDir(path string) error {
+	// no op for S3
+	return nil
+}
+
+// PutFile puts file into published storage at specified path
+func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
+	var (
+		source *os.File
+		err    error
+		fi     os.FileInfo
+	)
+	source, err = os.Open(sourceFilename)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	fi, err = source.Stat()
+	if err != nil {
+		return err
+	}
+
+	err = storage.bucket.PutReader(filepath.Join(storage.prefix, path), source, fi.Size(), "binary/octet-stream", storage.acl)
+	if err != nil {
+		return fmt.Errorf("error uploading %s to %s: %s", sourceFilename, storage, err)
+	}
+	return nil
+}
+
+// Remove removes single file under public path
+func (storage *PublishedStorage) Remove(path string) error {
+	err := storage.bucket.Del(filepath.Join(storage.prefix, path))
+	if err != nil {
+		return fmt.Errorf("error deleting %s from %s: %s", path, storage, err)
+	}
+	return nil
+}
+
+// RemoveDirs removes directory structure under public path
+func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
+	const page = 1000
+
+	filelist, err := storage.Filelist(path)
+	if err != nil {
+		return err
+	}
+
+	numParts := (len(filelist) + page - 1) / page
+
+	for i := 0; i < numParts; i++ {
+		var part []string
+		if i == numParts-1 {
+			part = filelist[i*page:]
+		} else {
+			part = filelist[i*page : (i+1)*page]
+		}
+		paths := make([]string, len(part))
+
+		for i := range part {
+			paths[i] = filepath.Join(storage.prefix, path, part[i])
+		}
+
+		err = storage.bucket.MultiDel(paths)
+		if err != nil {
+			return fmt.Errorf("error deleting multiple paths from %s: %s", storage, err)
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// LinkFromPool links package file from pool to dist's pool location
+//
+// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
+// sourcePool is instance of aptly.PackagePool
+// sourcePath is filepath to package file in package pool
+//
+// LinkFromPool returns relative path for the published file to be included in package index
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool,
+	sourcePath, sourceMD5 string, force bool) error {
+	// verify that package pool is local pool in filesystem
+	_ = sourcePool.(*files.PackagePool)
+
+	baseName := filepath.Base(sourcePath)
+	relPath := filepath.Join(publishedDirectory, baseName)
+	poolPath := filepath.Join(storage.prefix, relPath)
+
+	var (
+		dstKey *s3.Key
+		err    error
+	)
+
+	dstKey, err = storage.bucket.GetKey(poolPath)
+	if err != nil {
+		if s3err, ok := err.(*s3.Error); !ok || s3err.StatusCode != 404 {
+			return fmt.Errorf("error getting information about %s from %s: %s", poolPath, storage, err)
+		}
+	} else {
+		destinationMD5 := strings.Replace(dstKey.ETag, "\"", "", -1)
+		if destinationMD5 == sourceMD5 {
+			return nil
+		}
+
+		if !force && destinationMD5 != sourceMD5 {
+			return fmt.Errorf("error putting file to %s: file already exists and is different: %s", poolPath, storage)
+
+		}
+	}
+
+	return storage.PutFile(relPath, sourcePath)
+}
+
+// Filelist returns list of files under prefix
+func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
+	result := []string{}
+	marker := ""
+	prefix = filepath.Join(storage.prefix, prefix)
+	if prefix != "" {
+		prefix += "/"
+	}
+	for {
+		contents, err := storage.bucket.List(prefix, "", marker, 1000)
+		if err != nil {
+			return nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
+		}
+		lastKey := ""
+		for _, key := range contents.Contents {
+			if prefix == "" {
+				result = append(result, key.Key)
+			} else {
+				result = append(result, key.Key[len(prefix):])
+			}
+			lastKey = key.Key
+		}
+		if contents.IsTruncated {
+			marker = contents.NextMarker
+			if marker == "" {
+				// From the s3 docs: If response does not include the
+				// NextMarker and it is truncated, you can use the value of the
+				// last Key in the response as the marker in the subsequent
+				// request to get the next set of object keys.
+				marker = lastKey
+			}
+		} else {
+			break
+		}
+	}
+
+	return result, nil
+}
+
+// RenameFile renames (moves) file
+func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
+	err := storage.bucket.Copy(filepath.Join(storage.prefix, oldName), filepath.Join(storage.prefix, newName), storage.acl)
+	if err != nil {
+		return fmt.Errorf("error copying %s -> %s in %s: %s", oldName, newName, storage, err)
+	}
+
+	return storage.Remove(oldName)
+}

s3/public_test.go

@@ -0,0 +1,173 @@
+package s3
+
+import (
+	"github.com/mitchellh/goamz/aws"
+	"github.com/mitchellh/goamz/s3/s3test"
+	"github.com/smira/aptly/files"
+	"io/ioutil"
+	. "launchpad.net/gocheck"
+	"os"
+	"path/filepath"
+)
+
+type PublishedStorageSuite struct {
+	srv                      *s3test.Server
+	storage, prefixedStorage *PublishedStorage
+}
+
+var _ = Suite(&PublishedStorageSuite{})
+
+func (s *PublishedStorageSuite) SetUpTest(c *C) {
+	var err error
+	s.srv, err = s3test.NewServer(&s3test.Config{})
+	c.Assert(err, IsNil)
+	c.Assert(s.srv, NotNil)
+
+	auth, _ := aws.GetAuth("aa", "bb")
+	s.storage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "")
+	c.Assert(err, IsNil)
+
+	s.prefixedStorage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "lala")
+	c.Assert(err, IsNil)
+
+	err = s.storage.s3.Bucket("test").PutBucket("private")
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) TearDownTest(c *C) {
+	s.srv.Quit()
+}
+
+func (s *PublishedStorageSuite) TestNewPublishedStorage(c *C) {
+	stor, err := NewPublishedStorage("aa", "bbb", "", "", "", "")
+	c.Check(stor, IsNil)
+	c.Check(err, ErrorMatches, "unknown region: .*")
+}
+
+func (s *PublishedStorageSuite) TestPutFile(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to s3!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	data, err := s.storage.bucket.Get("a/b.txt")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+
+	err = s.prefixedStorage.PutFile("a/b.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	data, err = s.storage.bucket.Get("lala/a/b.txt")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+}
+
+func (s *PublishedStorageSuite) TestFilelist(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")
+		c.Check(err, IsNil)
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestRemove(c *C) {
+	err := s.storage.bucket.Put("a/b", []byte("test"), "binary/octet-stream", "private")
+	c.Check(err, IsNil)
+
+	err = s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	_, err = s.storage.bucket.Get("a/b")
+	c.Check(err, ErrorMatches, "The specified key does not exist.")
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
+	c.Skip("multiple-delete not available in s3test")
+
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")
+		c.Check(err, IsNil)
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
+
+}
+
+func (s *PublishedStorageSuite) TestRenameFile(c *C) {
+	c.Skip("copy not available in s3test")
+}
+
+func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
+	root := c.MkDir()
+	pool := files.NewPackagePool(root)
+
+	sourcePath := filepath.Join(root, "pool/c1/df/mars-invaders_1.03.deb")
+	err := os.MkdirAll(filepath.Dir(sourcePath), 0755)
+	c.Assert(err, IsNil)
+
+	err = ioutil.WriteFile(sourcePath, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+
+	sourcePath2 := filepath.Join(root, "pool/e9/df/mars-invaders_1.03.deb")
+	err = os.MkdirAll(filepath.Dir(sourcePath2), 0755)
+	c.Assert(err, IsNil)
+
+	err = ioutil.WriteFile(sourcePath2, []byte("Spam"), 0644)
+	c.Assert(err, IsNil)
+
+	// first link from pool
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "c1df1da7a1ce305a3b60af9d5733ac1d", false)
+	c.Check(err, IsNil)
+
+	data, err := s.storage.bucket.Get("pool/main/m/mars-invaders/mars-invaders_1.03.deb")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("Contents"))
+
+	// duplicate link from pool
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "c1df1da7a1ce305a3b60af9d5733ac1d", false)
+	c.Check(err, IsNil)
+
+	data, err = s.storage.bucket.Get("pool/main/m/mars-invaders/mars-invaders_1.03.deb")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath2, "e9dfd31cc505d51fc26975250750deab", false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
+
+	data, err = s.storage.bucket.Get("pool/main/m/mars-invaders/mars-invaders_1.03.deb")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict and force
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath2, "e9dfd31cc505d51fc26975250750deab", true)
+	c.Check(err, IsNil)
+
+	data, err = s.storage.bucket.Get("pool/main/m/mars-invaders/mars-invaders_1.03.deb")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("Spam"))
+}

s3/s3.go

@@ -0,0 +1,2 @@
+// Package s3 handles publishing to Amazon S3
+package s3

s3/s3_test.go

@@ -0,0 +1,11 @@
+package s3
+
+import (
+	. "launchpad.net/gocheck"
+	"testing"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

system/fixture.sh

@@ -27,3 +27,6 @@ aptly mirror update gnuplot-maverick
 
 aptly mirror create -with-sources gnuplot-maverick-src http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick
 aptly mirror update gnuplot-maverick-src
+
+aptly mirror create sensu http://repos.sensuapp.org/apt sensu
+aptly mirror update sensu

system/lib.py

@@ -66,6 +66,7 @@ class BaseTest(object):
     configFile = {
         "rootDir": "%s/.aptly" % os.environ["HOME"],
         "downloadConcurrency": 4,
+        "downloadSpeedLimit": 0,
         "architectures": [],
         "dependencyFollowSuggests": False,
         "dependencyFollowRecommends": False,
@@ -84,6 +85,8 @@ class BaseTest(object):
 
     outputMatchPrepare = None
 
+    captureResults = False
+
     def test(self):
         self.prepare()
         self.run()
@@ -181,15 +184,32 @@ class BaseTest(object):
     def expand_environ(self, gold):
         return string.Template(gold).substitute(os.environ)
 
+    def get_gold_filename(self, gold_name="gold"):
+        return os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), self.__class__.__name__ + "_" + gold_name)
+
     def get_gold(self, gold_name="gold"):
-        gold = os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), self.__class__.__name__ + "_" + gold_name)
-        return self.gold_processor(open(gold, "r").read())
+        return self.gold_processor(open(self.get_gold_filename(gold_name), "r").read())
 
     def check_output(self):
-        self.verify_match(self.get_gold(), self.output, match_prepare=self.outputMatchPrepare)
+        try:
+            self.verify_match(self.get_gold(), self.output, match_prepare=self.outputMatchPrepare)
+        except:
+            if self.captureResults:
+                with open(self.get_gold_filename(), "w") as f:
+                    f.write(self.output)
+            else:
+                raise
 
     def check_cmd_output(self, command, gold_name, match_prepare=None, expected_code=0):
-        self.verify_match(self.get_gold(gold_name), self.run_cmd(command, expected_code=expected_code), match_prepare)
+        try:
+            output = self.run_cmd(command, expected_code=expected_code)
+            self.verify_match(self.get_gold(gold_name), output, match_prepare)
+        except:
+            if self.captureResults:
+                with open(self.get_gold_filename(gold_name), "w") as f:
+                    f.write(output)
+            else:
+                raise
 
     def read_file(self, path):
         with open(os.path.join(os.environ["HOME"], ".aptly", path), "r") as f:
@@ -200,11 +220,26 @@ class BaseTest(object):
 
     def check_file_contents(self, path, gold_name, match_prepare=None):
         contents = self.read_file(path)
+        try:
 
-        self.verify_match(self.get_gold(gold_name), contents, match_prepare=match_prepare)
+            self.verify_match(self.get_gold(gold_name), contents, match_prepare=match_prepare)
+        except:
+            if self.captureResults:
+                with open(self.get_gold_filename(gold_name), "w") as f:
+                    f.write(contents)
+            else:
+                raise
 
     def check_file(self):
-        self.verify_match(self.get_gold(), open(self.checkedFile, "r").read())
+        contents = open(self.checkedFile, "r").read()
+        try:
+            self.verify_match(self.get_gold(), contents)
+        except:
+            if self.captureResults:
+                with open(self.get_gold_filename(), "w") as f:
+                    f.write(contents)
+            else:
+                raise
 
     def check_exists(self, path):
         if not os.path.exists(os.path.join(os.environ["HOME"], ".aptly", path)):

system/run.py

@@ -4,6 +4,7 @@ import glob
 import importlib
 import os
 import inspect
+import fnmatch
 import sys
 
 from lib import BaseTest
@@ -15,11 +16,11 @@ except ImportError:
         return s
 
 
-def run(include_long_tests=False, tests=None):
+def run(include_long_tests=False, capture_results=False, tests=None, filters=None):
     """
     Run system test.
     """
-    if tests is None:
+    if not tests:
         tests = glob.glob("t*_*")
     fails = []
     numTests = numFailed = numSkipped = 0
@@ -34,6 +35,17 @@ def run(include_long_tests=False, tests=None):
             if not (inspect.isclass(o) and issubclass(o, BaseTest) and o is not BaseTest):
                 continue
 
+            if filters:
+                matches = False
+
+                for filt in filters:
+                    if fnmatch.fnmatch(o.__name__, filt):
+                        matches = True
+                        break
+
+                if not matches:
+                    continue
+
             t = o()
             if t.longTest and not include_long_tests or not t.fixture_available():
                 numSkipped += 1
@@ -44,6 +56,7 @@ def run(include_long_tests=False, tests=None):
             sys.stdout.write("%s:%s... " % (test, o.__name__))
 
             try:
+                t.captureResults = capture_results
                 t.test()
             except BaseException, e:
                 numFailed += 1
@@ -69,10 +82,25 @@ def run(include_long_tests=False, tests=None):
 if __name__ == "__main__":
     os.chdir(os.path.realpath(os.path.dirname(sys.argv[0])))
     include_long_tests = False
+    capture_results = False
     tests = None
-    if len(sys.argv) > 1:
-        if sys.argv[1] == "--long":
+    args = sys.argv[1:]
+
+    while len(args) > 0 and args[0].startswith("--"):
+        if args[0] == "--long":
             include_long_tests = True
+        elif args[0] == "--capture":
+            capture_results = True
+
+        args = args[1:]
+
+    tests = []
+    filters = []
+
+    for arg in args:
+        if arg.startswith('t'):
+            tests.append(arg)
         else:
-            tests = sys.argv[1:]
-    run(include_long_tests, tests)
+            filters.append(arg)
+
+    run(include_long_tests, capture_results, tests, filters)

system/t01_version/VersionTest_gold

@@ -1 +1 @@
-aptly version: 0.5.1
+aptly version: 0.7.1

system/t02_config/CreateConfigTest_gold

@@ -1,6 +1,7 @@
 {
   "rootDir": "${HOME}/.aptly",
   "downloadConcurrency": 4,
+  "downloadSpeedLimit": 0,
   "architectures": [],
   "dependencyFollowSuggests": false,
   "dependencyFollowRecommends": false,
@@ -10,5 +11,6 @@
   "gpgDisableVerify": false,
   "downloadSourcePackages": false,
   "ppaDistributorID": "ubuntu",
-  "ppaCodename": ""
+  "ppaCodename": "",
+  "S3PublishEndpoints": {}
 }

system/t03_help/MainTest_gold

@@ -21,3 +21,4 @@ Options:
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests
+ERROR: unable to parse command

system/t03_help/MirrorCreateHelpTest_gold

@@ -19,6 +19,8 @@ Options:
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests
+  -filter="": filter packages in mirror
+  -filter-with-deps=false: when filtering, include dependencies of matching packages as well
   -ignore-signatures=false: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
   -with-sources=false: download source packages in addition to binary packages

system/t03_help/MirrorCreateTest_gold

@@ -10,6 +10,9 @@ Options:
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests
+  -filter="": filter packages in mirror
+  -filter-with-deps=false: when filtering, include dependencies of matching packages as well
   -ignore-signatures=false: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
   -with-sources=false: download source packages in addition to binary packages
+ERROR: unable to parse command

system/t03_help/MirrorHelpTest_gold

@@ -4,7 +4,9 @@ Commands:
 
     create      create new mirror
     drop        delete mirror
+    edit        edit properties of mirorr
     list        list mirrors
+    rename      renames mirror
     show        show details about mirror
     update      update mirror
 

system/t03_help/MirrorTest_gold

@@ -4,7 +4,9 @@ Commands:
 
     create      create new mirror
     drop        delete mirror
+    edit        edit properties of mirorr
     list        list mirrors
+    rename      renames mirror
     show        show details about mirror
     update      update mirror
 
@@ -18,3 +20,4 @@ Options:
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests
+ERROR: unable to parse command

system/t03_help/WrongFlagTest_gold

@@ -0,0 +1,19 @@
+flag provided but not defined: -fxz
+Usage: aptly mirror create <name> <archive url> <distribution> [<component1> ...]
+
+aptly mirror create - create new mirror
+
+
+Options:
+  -architectures="": list of architectures to consider during (comma-separated), default to all available
+  -config="": location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)
+  -dep-follow-all-variants=false: when processing dependencies, follow a & b if depdency is 'a|b'
+  -dep-follow-recommends=false: when processing dependencies, follow Recommends
+  -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
+  -dep-follow-suggests=false: when processing dependencies, follow Suggests
+  -filter="": filter packages in mirror
+  -filter-with-deps=false: when filtering, include dependencies of matching packages as well
+  -ignore-signatures=false: disable verification of Release file signatures
+  -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
+  -with-sources=false: download source packages in addition to binary packages
+ERROR: unable to parse flags

system/t03_help/__init__.py

@@ -10,6 +10,7 @@ class MainTest(BaseTest):
     """
     main
     """
+    expectedCode = 2
     runCmd = "aptly"
 
     outputMatchPrepare = lambda _, s: re.sub(r'  -(cpuprofile|memprofile|memstats|meminterval)=.*\n', '', s, flags=re.MULTILINE)
@@ -19,6 +20,7 @@ class MirrorTest(BaseTest):
     """
     main
     """
+    expectedCode = 2
     runCmd = "aptly mirror"
 
 
@@ -26,6 +28,7 @@ class MirrorCreateTest(BaseTest):
     """
     main
     """
+    expectedCode = 2
     runCmd = "aptly mirror create"
 
 
@@ -50,3 +53,11 @@ class MirrorCreateHelpTest(BaseTest):
     main
     """
     runCmd = "aptly help mirror create"
+
+
+class WrongFlagTest(BaseTest):
+    """
+    main
+    """
+    expectedCode = 2
+    runCmd = "aptly mirror create -fxz=sss"

system/t04_mirror/CreateMirror11Test_mirror_show

@@ -10,10 +10,10 @@ Information from release file:
 Architectures: amd64 armel i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 sparc
 Codename: squeeze
 Components: main contrib non-free
-Date: Sat, 15 Feb 2014 10:01:30 UTC
-Description:  Debian 6.0.9 Released 15 February 2014
+Date: Sat, 19 Jul 2014 11:02:06 UTC
+Description:  Debian 6.0.10 Released 19 July 2014
 
 Label: Debian
 Origin: Debian
 Suite: oldstable
-Version: 6.0.9
+Version: 6.0.10

system/t04_mirror/CreateMirror13Test_mirror_show

@@ -10,10 +10,10 @@ Information from release file:
 Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
 Codename: wheezy
 Components: main contrib non-free
-Date: Sat, 26 Apr 2014 09:27:11 UTC
-Description:  Debian 7.5 Released 26 April 2014
+Date: Sat, 12 Jul 2014 10:59:25 UTC
+Description:  Debian 7.6 Released 12 July 2014
 
 Label: Debian
 Origin: Debian
 Suite: stable
-Version: 7.5
+Version: 7.6

system/t04_mirror/CreateMirror17Test_mirror_show

@@ -10,10 +10,10 @@ Information from release file:
 Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
 Codename: wheezy
 Components: main contrib non-free
-Date: Sat, 26 Apr 2014 09:27:11 UTC
-Description:  Debian 7.5 Released 26 April 2014
+Date: Sat, 12 Jul 2014 10:59:25 UTC
+Description:  Debian 7.6 Released 12 July 2014
 
 Label: Debian
 Origin: Debian
 Suite: stable
-Version: 7.5
+Version: 7.6

system/t04_mirror/CreateMirror1Test_mirror_show

@@ -10,10 +10,10 @@ Information from release file:
 Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
 Codename: wheezy
 Components: main contrib non-free
-Date: Sat, 26 Apr 2014 09:27:11 UTC
-Description:  Debian 7.5 Released 26 April 2014
+Date: Sat, 12 Jul 2014 10:59:25 UTC
+Description:  Debian 7.6 Released 12 July 2014
 
 Label: Debian
 Origin: Debian
 Suite: stable
-Version: 7.5
+Version: 7.6

system/t04_mirror/CreateMirror21Test_mirror_show

@@ -8,6 +8,6 @@ Last update: never
 
 Information from release file:
 Architectures: all
-Date: Thu, 01 May 2014 05:51:11 UTC
+Date: Wed, 30 Jul 2014 15:23:01 UTC
 Origin: jenkins-ci.org
 Suite: binary

system/t04_mirror/CreateMirror22Test_gold

@@ -0,0 +1,4 @@
+Downloading http://security.debian.org/dists/wheezy/updates/Release...
+
+Mirror [mirror22]: http://security.debian.org/ wheezy/updates successfully added.
+You can run 'aptly mirror update mirror22' to download repository contents.