Disable support for aptly task for now. #96

Config option should act as default, while flag should override it only if set.

.travis.yml

@@ -1,14 +1,17 @@
 language: go
 
 go:
-  - 1.1
   - 1.2.1
+  - 1.3.1
   - tip
 
 env:
   global:
     - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
-
+    - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -y python-boto
 install:
   - make prepare
 

AUTHORS

@@ -0,0 +1,7 @@
+List of contributors, in chronological order:
+
+* Andrey Smirnov (https://github.com/smira)
+* Sebastien Binet (https://github.com/sbinet)
+* Ryan Uber (https://github.com/ryanuber)
+* Simon Aquino (https://github.com/simonaquino)
+* Vincent Batoufflet (https://github.com/vbatoufflet)

Gomfile

@@ -1,12 +1,17 @@
 gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
 gom 'code.google.com/p/go.crypto/ssh/terminal', :commit => '7aa593ce8cea'
 gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2'
+gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556'
 gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
 gom 'github.com/cheggaaa/pb', :commit => '74be7a1388046f374ac36e93d46f5d56e856f827'
-gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
-gom 'github.com/smira/flag', :commit => '0d0aac2addb39050f45e92c5a6252926096dc841'
+gom 'github.com/vaughan0/go-ini', :commit => 'a98ad7ee00ec53921f08832bc06ecf7fd600e6a1'
+gom 'github.com/mattn/go-shellwords', :commit => 'c7ca6f94add751566a61cf2199e1de78d4c3eee4'
+gom 'github.com/mitchellh/goamz/s3', :commit => 'e7664b32019f31fd1bdf33f9e85f28722f700405'
 gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => 'ff3719c6816e2cd194f05058452d660608e178ac'
+gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
+gom 'github.com/smira/flag', :commit => '357ed3e599ffcbd4aeaa828e1d10da2df3ea5107'
+gom 'github.com/smira/go-ftp-protocol/protocol', :commit => '066b75c2b70dca7ae10b1b88b47534a3c31ccfaa'
+gom 'github.com/syndtr/goleveldb/leveldb', :commit => 'e2fa4e6ac1cc41a73bc9fd467878ecbf65df5cc3'
 gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
 gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
 

Makefile

@@ -1,6 +1,6 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-PACKAGES=database deb files http utils
-ALL_PACKAGES=aptly cmd console database deb files http utils
+PACKAGES=database deb files http query s3 utils
+ALL_PACKAGES=aptly cmd console database deb files http query s3 utils
 BINPATH=$(abspath ./_vendor/bin)
 GOM_ENVIRONMENT=-test
 PYTHON?=python
@@ -43,9 +43,7 @@ install:
 	$(GOM) build -o $(BINPATH)/aptly
 
 system-test: install
-ifeq ($(GOVERSION),$(filter $(GOVERSION),go1.2 go1.2.1 devel))
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
-endif
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
 	PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long
 
@@ -69,7 +67,7 @@ package:
 	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
 	gzip root/usr/share/man/man1/aptly.1
 	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
-	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" -C root/ .
+	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 -C root/ .
 	mv aptly_$(VERSION)_*.deb ~
 
 src-package:
@@ -77,11 +75,10 @@ src-package:
 	mkdir -p aptly-$(VERSION)/src/github.com/smira/aptly/
 	cd aptly-$(VERSION)/src/github.com/smira/ && git clone https://github.com/smira/aptly && cd aptly && git checkout v$(VERSION)
 	cd aptly-$(VERSION)/src/github.com/smira/aptly && gom -production install
-	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . -name .git -print | xargs rm -rf
-	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . -name .bzr -print | xargs rm -rf
-	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . -name .hg -print | xargs rm -rf
+	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . \( -name .git -o -name .bzr -o -name .hg \) -print | xargs rm -rf
 	rm -rf aptly-$(VERSION)/src/github.com/smira/aptly/_vendor/{pkg,bin}
 	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
 	rm -rf aptly-$(VERSION)
+	curl -T aptly-$(VERSION)-src.tar.bz2 -usmira:$(BINTRAY_KEY) https://api.bintray.com/content/smira/aptly/aptly/$(VERSION)/$(VERSION)/aptly-$(VERSION)-src.tar.bz2
 
 .PHONY: coverage.out

README.rst

@@ -8,8 +8,14 @@ aptly
 .. image:: https://coveralls.io/repos/smira/aptly/badge.png?branch=HEAD
     :target: https://coveralls.io/r/smira/aptly?branch=HEAD
 
+.. image:: http://gobuild.io/badge/github.com/smira/aptly/download.png
+    :target: http://gobuild.io/github.com/smira/aptly
+
 Aptly is a swiss army knife for Debian repository management.
 
+.. image:: http://www.aptly.info/img/aptly_logo.png
+    :target: http://www.aptly.info/
+
 Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support use
 mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
 
@@ -20,14 +26,14 @@ Aptly features: ("+" means planned features)
 * publish snapshot as Debian repository, ready to be consumed by apt
 * controlled update of one or more packages in snapshot from upstream mirror, tracking dependencies
 * merge two or more snapshots into one
-* filter repository by search query, pulling dependencies when required (+)
-* publish self-made packages as Debian repositories (+)
+* filter repository by search query, pulling dependencies when required
+* publish self-made packages as Debian repositories
 * mirror repositories "as-is" (without resigning with user's key) (+)
 * support for yum repositories (+)
 
 Current limitations:
 
-* debian-installer and translations not supported yet
+* translations are not supported yet
 
 Download
 --------
@@ -51,7 +57,7 @@ Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
 
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.1+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.2+ required)::
 
     go get -u github.com/mattn/gom
     mkdir -p $GOPATH/src/github.com/smira/aptly

aptly/interfaces.go

@@ -5,7 +5,6 @@ package aptly
 import (
 	"github.com/smira/aptly/utils"
 	"io"
-	"os"
 )
 
 // PackagePool is asbtraction of package pool storage.
@@ -26,26 +25,34 @@ type PackagePool interface {
 
 // PublishedStorage is abstraction of filesystem storing all published repositories
 type PublishedStorage interface {
-	// PublicPath returns root of public part
-	PublicPath() string
 	// MkDir creates directory recursively under public path
 	MkDir(path string) error
-	// CreateFile creates file for writing under public path
-	CreateFile(path string) (*os.File, error)
+	// PutFile puts file into published storage at specified path
+	PutFile(path string, sourceFilename string) error
 	// RemoveDirs removes directory structure under public path
 	RemoveDirs(path string, progress Progress) error
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory string, sourcePool PackagePool, sourcePath string) error
+	LinkFromPool(publishedDirectory string, sourcePool PackagePool, sourcePath, sourceMD5 string, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
-	// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
-	ChecksumsForFile(path string) (utils.ChecksumInfo, error)
 	// RenameFile renames (moves) file
 	RenameFile(oldName, newName string) error
 }
 
+// LocalPublishedStorage is published storage on local filesystem
+type LocalPublishedStorage interface {
+	// PublicPath returns root of public part
+	PublicPath() string
+}
+
+// PublishedStorageProvider is a thing that returns PublishedStorage by name
+type PublishedStorageProvider interface {
+	// GetPublishedStorage returns PublishedStorage by name
+	GetPublishedStorage(name string) PublishedStorage
+}
+
 // Progress is a progress displaying entity, it allows progress bars & simple prints
 type Progress interface {
 	// Writer interface to support progress bar ticking
@@ -83,6 +90,8 @@ type Downloader interface {
 	// Shutdown stops downloader after current tasks are finished,
 	// but doesn't process rest of queue
 	Shutdown()
+	// Abort stops downloader without waiting for shutdown
+	Abort()
 	// GetProgress returns Progress object
 	GetProgress() Progress
 }

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
 // Version of aptly
-const Version = "0.6"
+const Version = "0.8"
 
 // Enable debugging features?
 const EnableDebug = false

cmd/cmd.go

@@ -34,6 +34,18 @@ func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 	return
 }
 
+// LookupOption checks boolean flag with default (usually config) and command-line
+// setting
+func LookupOption(defaultValue bool, flags *flag.FlagSet, name string) (result bool) {
+	result = defaultValue
+
+	if flags.IsSet(name) {
+		result = flags.Lookup(name).Value.Get().(bool)
+	}
+
+	return
+}
+
 // RootCommand creates root command in command tree
 func RootCommand() *commander.Command {
 	cmd := &commander.Command{
@@ -46,9 +58,9 @@ upgrade individual packages, take snapshots and publish them
 back as Debian repositories.
 
 aptly's goal is to establish repeatability and controlled changes
-in a package-centric environment. aptly allows to fix a set of packages
+in a package-centric environment. aptly allows one to fix a set of packages
 in a repository, so that package installation and upgrade becomes
-deterministic. At the same time aptly allows to perform controlled,
+deterministic. At the same time aptly allows one to perform controlled,
 fine-grained changes in repository contents to transition your
 package environment to new version.`,
 		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
@@ -59,8 +71,11 @@ package environment to new version.`,
 			makeCmdRepo(),
 			makeCmdServe(),
 			makeCmdSnapshot(),
+			// Disabled on no docs
+			//makeCmdTask(),
 			makeCmdPublish(),
 			makeCmdVersion(),
+			makeCmdPackage(),
 		},
 	}
 

cmd/context.go

@@ -8,6 +8,7 @@ import (
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/s3"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -21,14 +22,14 @@ import (
 
 // AptlyContext is a common context shared by all commands
 type AptlyContext struct {
-	flags        *flag.FlagSet
-	configLoaded bool
+	flags, globalFlags *flag.FlagSet
+	configLoaded       bool
 
 	progress          aptly.Progress
 	downloader        aptly.Downloader
 	database          database.Storage
 	packagePool       aptly.PackagePool
-	publishedStorage  aptly.PublishedStorage
+	publishedStorages map[string]aptly.PublishedStorage
 	collectionFactory *deb.CollectionFactory
 	dependencyOptions int
 	architecturesList []string
@@ -40,6 +41,9 @@ type AptlyContext struct {
 
 var context *AptlyContext
 
+// Check interface
+var _ aptly.PublishedStorageProvider = &AptlyContext{}
+
 // FatalError is type for panicking to abort execution with non-zero
 // exit code and print meaningful explanation
 type FatalError struct {
@@ -61,7 +65,7 @@ func (context *AptlyContext) Config() *utils.ConfigStructure {
 	if !context.configLoaded {
 		var err error
 
-		configLocation := context.flags.Lookup("config").Value.String()
+		configLocation := context.globalFlags.Lookup("config").Value.String()
 		if configLocation != "" {
 			err = utils.LoadConfig(configLocation, &utils.Config)
 
@@ -100,16 +104,16 @@ func (context *AptlyContext) Config() *utils.ConfigStructure {
 func (context *AptlyContext) DependencyOptions() int {
 	if context.dependencyOptions == -1 {
 		context.dependencyOptions = 0
-		if context.Config().DepFollowSuggests || context.flags.Lookup("dep-follow-suggests").Value.Get().(bool) {
+		if LookupOption(context.Config().DepFollowSuggests, context.globalFlags, "dep-follow-suggests") {
 			context.dependencyOptions |= deb.DepFollowSuggests
 		}
-		if context.Config().DepFollowRecommends || context.flags.Lookup("dep-follow-recommends").Value.Get().(bool) {
+		if LookupOption(context.Config().DepFollowRecommends, context.globalFlags, "dep-follow-recommends") {
 			context.dependencyOptions |= deb.DepFollowRecommends
 		}
-		if context.Config().DepFollowAllVariants || context.flags.Lookup("dep-follow-all-variants").Value.Get().(bool) {
+		if LookupOption(context.Config().DepFollowAllVariants, context.globalFlags, "dep-follow-all-variants") {
 			context.dependencyOptions |= deb.DepFollowAllVariants
 		}
-		if context.Config().DepFollowSource || context.flags.Lookup("dep-follow-source").Value.Get().(bool) {
+		if LookupOption(context.Config().DepFollowSource, context.globalFlags, "dep-follow-source") {
 			context.dependencyOptions |= deb.DepFollowSource
 		}
 	}
@@ -121,7 +125,7 @@ func (context *AptlyContext) DependencyOptions() int {
 func (context *AptlyContext) ArchitecturesList() []string {
 	if context.architecturesList == nil {
 		context.architecturesList = context.Config().Architectures
-		optionArchitectures := context.flags.Lookup("architectures").Value.String()
+		optionArchitectures := context.globalFlags.Lookup("architectures").Value.String()
 		if optionArchitectures != "" {
 			context.architecturesList = strings.Split(optionArchitectures, ",")
 		}
@@ -143,7 +147,16 @@ func (context *AptlyContext) Progress() aptly.Progress {
 // Downloader returns instance of current downloader
 func (context *AptlyContext) Downloader() aptly.Downloader {
 	if context.downloader == nil {
-		context.downloader = http.NewDownloader(context.Config().DownloadConcurrency, context.Progress())
+		var downloadLimit int64
+		limitFlag := context.flags.Lookup("download-limit")
+		if limitFlag != nil {
+			downloadLimit = limitFlag.Value.Get().(int64)
+		}
+		if downloadLimit == 0 {
+			downloadLimit = context.Config().DownloadLimit
+		}
+		context.downloader = http.NewDownloader(context.Config().DownloadConcurrency,
+			downloadLimit*1024, context.Progress())
 	}
 
 	return context.downloader
@@ -168,6 +181,36 @@ func (context *AptlyContext) Database() (database.Storage, error) {
 	return context.database, nil
 }
 
+// CloseDatabase closes the db temporarily
+func (context *AptlyContext) CloseDatabase() error {
+	if context.database == nil {
+		return nil
+	}
+
+	return context.database.Close()
+}
+
+// ReOpenDatabase reopens the db after close
+func (context *AptlyContext) ReOpenDatabase() error {
+	if context.database == nil {
+		return nil
+	}
+
+	const MaxTries = 10
+	const Delay = 10 * time.Second
+
+	for try := 0; try < MaxTries; try++ {
+		err := context.database.ReOpen()
+		if err == nil || strings.Index(err.Error(), "resource temporarily unavailable") == -1 {
+			return err
+		}
+		context.Progress().Printf("Unable to reopen database, sleeping %s\n", Delay)
+		<-time.After(Delay)
+	}
+
+	return fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
+}
+
 // CollectionFactory builds factory producing all kinds of collections
 func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
 	if context.collectionFactory == nil {
@@ -190,13 +233,37 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
 	return context.packagePool
 }
 
-// PublishedStorage returns instance of PublishedStorage
-func (context *AptlyContext) PublishedStorage() aptly.PublishedStorage {
-	if context.publishedStorage == nil {
-		context.publishedStorage = files.NewPublishedStorage(context.Config().RootDir)
+// GetPublishedStorage returns instance of PublishedStorage
+func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
+	publishedStorage, ok := context.publishedStorages[name]
+	if !ok {
+		if name == "" {
+			publishedStorage = files.NewPublishedStorage(context.Config().RootDir)
+		} else if strings.HasPrefix(name, "s3:") {
+			params, ok := context.Config().S3PublishRoots[name[3:]]
+			if !ok {
+				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
+			}
+
+			var err error
+			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
+				params.Region, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
+				params.EncryptionMethod, params.PlusWorkaround)
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			Fatal(fmt.Errorf("unknown published storage format: %v", name))
+		}
+		context.publishedStorages[name] = publishedStorage
 	}
 
-	return context.publishedStorage
+	return publishedStorage
+}
+
+// UpdateFlags sets internal copy of flags in the context
+func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) {
+	context.flags = flags
 }
 
 // ShutdownContext shuts context down
@@ -219,12 +286,27 @@ func ShutdownContext() {
 	}
 	if context.database != nil {
 		context.database.Close()
+		context.database = nil
 	}
 	if context.downloader != nil {
-		context.downloader.Shutdown()
+		context.downloader.Abort()
+		context.downloader = nil
 	}
 	if context.progress != nil {
 		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
+// CleanupContext does partial shutdown of context
+func CleanupContext() {
+	if context.downloader != nil {
+		context.downloader.Shutdown()
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
 	}
 }
 
@@ -232,7 +314,16 @@ func ShutdownContext() {
 func InitContext(flags *flag.FlagSet) error {
 	var err error
 
-	context = &AptlyContext{flags: flags, dependencyOptions: -1}
+	if context != nil {
+		panic("context already initialized")
+	}
+
+	context = &AptlyContext{
+		flags:             flags,
+		globalFlags:       flags,
+		dependencyOptions: -1,
+		publishedStorages: map[string]aptly.PublishedStorage{},
+	}
 
 	if aptly.EnableDebug {
 		cpuprofile := flags.Lookup("cpuprofile").Value.String()

cmd/mirror.go

@@ -8,7 +8,7 @@ import (
 )
 
 func getVerifier(flags *flag.FlagSet) (utils.Verifier, error) {
-	if context.Config().GpgDisableVerify || flags.Lookup("ignore-signatures").Value.Get().(bool) {
+	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
 		return nil, nil
 	}
 
@@ -54,6 +54,9 @@ func makeCmdMirror() *commander.Command {
 			makeCmdMirrorShow(),
 			makeCmdMirrorDrop(),
 			makeCmdMirrorUpdate(),
+			makeCmdMirrorRename(),
+			makeCmdMirrorEdit(),
+			makeCmdMirrorSearch(),
 		},
 	}
 }

cmd/mirror_create.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"strings"
@@ -15,7 +16,8 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	downloadSources := context.Config().DownloadSourcePackages || context.flags.Lookup("with-sources").Value.Get().(bool)
+	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.flags, "with-sources")
+	downloadUdebs := context.flags.Lookup("with-udebs").Value.Get().(bool)
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -32,11 +34,22 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		archiveURL, distribution, components = args[1], args[2], args[3:]
 	}
 
-	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(), downloadSources)
+	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
+		downloadSources, downloadUdebs)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
+	repo.Filter = context.flags.Lookup("filter").Value.String()
+	repo.FilterWithDeps = context.flags.Lookup("filter-with-deps").Value.Get().(bool)
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to create mirror: %s", err)
+		}
+	}
+
 	verifier, err := getVerifier(context.flags)
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
@@ -63,7 +76,7 @@ func makeCmdMirrorCreate() *commander.Command {
 		Short:     "create new mirror",
 		Long: `
 Creates mirror <name> of remote repository, aptly supports both regular and flat Debian repositories exported
-via HTTP. aptly would try download Release file from remote repository and verify its' signature. Command
+via HTTP and FTP. aptly would try download Release file from remote repository and verify its' signature. Command
 line format resembles apt utlitily sources.list(5).
 
 PPA urls could specified in short format:
@@ -79,6 +92,9 @@ Example:
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/mirror_drop.go

@@ -20,6 +20,11 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
 	force := context.flags.Lookup("force").Value.Get().(bool)
 	if !force {
 		snapshots := context.CollectionFactory().SnapshotCollection().ByRemoteRepoSource(repo)

cmd/mirror_edit.go

@@ -0,0 +1,91 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	context.flags.Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "filter":
+			repo.Filter = flag.Value.String()
+		case "filter-with-deps":
+			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-sources":
+			repo.DownloadSources = flag.Value.Get().(bool)
+		case "with-udebs":
+			repo.DownloadUdebs = flag.Value.Get().(bool)
+		}
+	})
+
+	if repo.IsFlat() && repo.DownloadUdebs {
+		return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
+	}
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
+	if context.globalFlags.Lookup("architectures").Value.String() != "" {
+		repo.Architectures = context.ArchitecturesList()
+
+		err = repo.Fetch(context.Downloader(), nil)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fmt.Printf("Mirror %s successfully updated.\n", repo)
+	return err
+}
+
+func makeCmdMirrorEdit() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorEdit,
+		UsageLine: "edit <name>",
+		Short:     "edit mirror settings",
+		Long: `
+Command edit allows one to change settings of mirror:
+filters, list of architectures.
+
+Example:
+
+  $ aptly mirror edit -filter=nginx -filter-with-deps some-mirror
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-edit", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+
+	return cmd
+}

cmd/mirror_rename.go

@@ -0,0 +1,64 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyMirrorRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.RemoteRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().RemoteRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: mirror %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nMirror %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdMirrorRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames mirror",
+		Long: `
+Command changes name of the mirror.Mirror name should be unique.
+
+Example:
+
+  $ aptly mirror rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/mirror_search.go

@@ -0,0 +1,26 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdMirrorSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> <package-query>",
+		Short:     "search mirror for packages matching query",
+		Long: `
+Command search displays list of packages in mirror that match package query
+
+Example:
+
+    $ aptly mirror search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+
+	return cmd
+}

cmd/mirror_show.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -28,6 +29,9 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 	}
 
 	fmt.Printf("Name: %s\n", repo.Name)
+	if repo.Status == deb.MirrorUpdating {
+		fmt.Printf("Status: In Update (PID %d)\n", repo.WorkerPID)
+	}
 	fmt.Printf("Archive Root URL: %s\n", repo.ArchiveRoot)
 	fmt.Printf("Distribution: %s\n", repo.Distribution)
 	fmt.Printf("Components: %s\n", strings.Join(repo.Components, ", "))
@@ -37,6 +41,19 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		downloadSources = "yes"
 	}
 	fmt.Printf("Download Sources: %s\n", downloadSources)
+	downloadUdebs := "no"
+	if repo.DownloadUdebs {
+		downloadUdebs = "yes"
+	}
+	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
+	if repo.Filter != "" {
+		fmt.Printf("Filter: %s\n", repo.Filter)
+		filterWithDeps := "no"
+		if repo.FilterWithDeps {
+			filterWithDeps = "yes"
+		}
+		fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
+	}
 	if repo.LastDownloadDate.IsZero() {
 		fmt.Printf("Last update: never\n")
 	} else {

cmd/mirror_update.go

@@ -2,8 +2,14 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
+	"os"
+	"os/signal"
+	"strings"
 )
 
 func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
@@ -25,6 +31,14 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
+	force := context.flags.Lookup("force").Value.Get().(bool)
+	if !force {
+		err = repo.CheckLock()
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
+
 	ignoreMismatch := context.flags.Lookup("ignore-checksums").Value.Get().(bool)
 
 	verifier, err := getVerifier(context.flags)
@@ -37,11 +51,112 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = repo.Download(context.Progress(), context.Downloader(), context.CollectionFactory(), context.PackagePool(), ignoreMismatch)
+	context.Progress().Printf("Downloading & parsing package files...\n")
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
+	if repo.Filter != "" {
+		context.Progress().Printf("Applying filter...\n")
+		var filterQuery deb.PackageQuery
+
+		filterQuery, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+
+		var oldLen, newLen int
+		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+		context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen)
+	}
+
+	var (
+		downloadSize int64
+		queue        []deb.PackageDownloadTask
+	)
+
+	context.Progress().Printf("Building download queue...\n")
+	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	defer func() {
+		// on any interruption, unlock the mirror
+		err := context.ReOpenDatabase()
+		if err == nil {
+			repo.MarkAsIdle()
+			context.CollectionFactory().RemoteRepoCollection().Update(repo)
+		}
+	}()
+
+	repo.MarkAsUpdating()
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = context.CloseDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	// Catch ^C
+	sigch := make(chan os.Signal)
+	signal.Notify(sigch, os.Interrupt)
+
+	count := len(queue)
+	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
+
+	// Download from the queue
+	context.Progress().InitBar(downloadSize, true)
+
+	// Download all package files
+	ch := make(chan error, count)
+
+	// In separate goroutine (to avoid blocking main), push queue to downloader
+	go func() {
+		for _, task := range queue {
+			context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
+		}
+
+		// We don't need queue after this point
+		queue = nil
+	}()
+
+	// Wait for all downloads to finish
+	errors := make([]string, 0)
+
+	for count > 0 {
+		select {
+		case <-sigch:
+			signal.Stop(sigch)
+			return fmt.Errorf("unable to update: interrupted")
+		case err = <-ch:
+			if err != nil {
+				errors = append(errors, err.Error())
+			}
+			count--
+		}
+	}
+
+	context.Progress().ShutdownBar()
+	signal.Stop(sigch)
+
+	if len(errors) > 0 {
+		return fmt.Errorf("unable to update: download errors:\n  %s\n", strings.Join(errors, "\n  "))
+	}
+
+	err = context.ReOpenDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	repo.FinalizeDownload()
 	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -68,8 +183,10 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-update", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("force", false, "force update mirror even if it is locked by another process")
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/package.go

@@ -0,0 +1,16 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdPackage() *commander.Command {
+	return &commander.Command{
+		UsageLine: "package",
+		Short:     "operations on packages",
+		Subcommands: []*commander.Command{
+			makeCmdPackageSearch(),
+			makeCmdPackageShow(),
+		},
+	}
+}

cmd/package_search.go

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPackageSearch(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	q, err := query.Parse(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	result := q.Query(context.CollectionFactory().PackageCollection())
+	result.ForEach(func(p *deb.Package) error {
+		context.Progress().Printf("%s\n", p)
+		return nil
+	})
+
+	return err
+}
+
+func makeCmdPackageSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageSearch,
+		UsageLine: "search <package-query>",
+		Short:     "search for packages matching query",
+		Long: `
+Command search displays list of packages in whole DB that match package query
+
+Example:
+
+    $ aptly package search '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-search", flag.ExitOnError),
+	}
+
+	return cmd
+}

cmd/package_show.go

@@ -0,0 +1,137 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"os"
+)
+
+func printReferencesTo(p *deb.Package) (err error) {
+	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  mirror %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  local repo %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+		if snapshot.RefList().Has(p) {
+			fmt.Printf("  snapshot %s\n", snapshot)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func aptlyPackageShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	q, err := query.Parse(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	withFiles := context.flags.Lookup("with-files").Value.Get().(bool)
+	withReferences := context.flags.Lookup("with-references").Value.Get().(bool)
+
+	w := bufio.NewWriter(os.Stdout)
+
+	result := q.Query(context.CollectionFactory().PackageCollection())
+
+	err = result.ForEach(func(p *deb.Package) error {
+		p.Stanza().WriteTo(w)
+		w.Flush()
+		fmt.Printf("\n")
+
+		if withFiles {
+			fmt.Printf("Files in the pool:\n")
+			for _, f := range p.Files() {
+				path, err := context.PackagePool().Path(f.Filename, f.Checksums.MD5)
+				if err != nil {
+					return err
+				}
+				fmt.Printf("  %s\n", path)
+			}
+			fmt.Printf("\n")
+		}
+
+		if withReferences {
+			fmt.Printf("References to package:\n")
+			printReferencesTo(p)
+			fmt.Printf("\n")
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdPackageShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageShow,
+		UsageLine: "show <package-query>",
+		Short:     "show details about packages matcing query",
+		Long: `
+Command shows displays detailed meta-information about packages
+matching query. Information from Debian control file is displayed.
+Optionally information about package files and
+inclusion into mirrors/snapshots/local repos is shown.
+
+Example:
+
+    $ aptly package show nginx-light_1.2.1-2.2+wheezy2_i386'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-files", false, "display information about files from package pool")
+	cmd.Flag.Bool("with-references", false, "display information about mirrors, snapshots and local repos referencing this package")
+
+	return cmd
+}

cmd/publish.go

@@ -4,16 +4,18 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
+	"strings"
 )
 
 func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
-	if flags.Lookup("skip-signing").Value.Get().(bool) || context.Config().GpgDisableSign {
+	if LookupOption(context.Config().GpgDisableSign, flags, "skip-signing") {
 		return nil, nil
 	}
 
 	signer := &utils.GpgSigner{}
 	signer.SetKey(flags.Lookup("gpg-key").Value.String())
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
+	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
 
 	err := signer.Init()
 	if err != nil {
@@ -24,6 +26,20 @@ func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
 
 }
 
+func parsePrefix(param string) (storage, prefix string) {
+	i := strings.LastIndex(param, ":")
+	if i != -1 {
+		storage = param[:i]
+		prefix = param[i+1:]
+		if prefix == "" {
+			prefix = "."
+		}
+	} else {
+		prefix = param
+	}
+	return
+}
+
 func makeCmdPublish() *commander.Command {
 	return &commander.Command{
 		UsageLine: "publish",

cmd/publish_drop.go

@@ -13,13 +13,15 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	if len(args) == 2 {
-		prefix = args[1]
+		param = args[1]
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Remove(context.PublishedStorage(), prefix, distribution,
+	storage, prefix := parsePrefix(param)
+
+	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
 		context.CollectionFactory(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
@@ -33,11 +35,11 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 func makeCmdPublishDrop() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishDrop,
-		UsageLine: "drop <distribution> [<prefix>]",
+		UsageLine: "drop <distribution> [[<endpoint>:]<prefix>]",
 		Short:     "remove published repository",
 		Long: `
-Command removes whatever has been published under specified <prefix> and
-<distribution> name.
+Command removes whatever has been published under specified <prefix>,
+publishing <endpoint> and <distribution> name.
 
 Example:
 

cmd/publish_list.go

@@ -25,7 +25,7 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 		}
 
 		if raw {
-			published = append(published, fmt.Sprintf("%s %s", repo.Prefix, repo.Distribution))
+			published = append(published, fmt.Sprintf("%s %s", repo.StoragePrefix(), repo.Distribution))
 		} else {
 			published = append(published, repo.String())
 		}

cmd/publish_repo.go

@@ -8,7 +8,7 @@ import (
 func makeCmdPublishRepo() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSnapshotOrRepo,
-		UsageLine: "repo <name> [<prefix>]",
+		UsageLine: "repo <name> [[<endpoint>:]<prefix>]",
 		Short:     "publish local repository",
 		Long: `
 Command publishes current state of local repository ready to be consumed
@@ -37,9 +37,12 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd
 }

cmd/publish_snapshot.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
@@ -19,13 +20,14 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	var prefix string
+	var param string
 	if len(args) == len(components)+1 {
-		prefix = args[len(components)]
+		param = args[len(components)]
 		args = args[0 : len(args)-1]
 	} else {
-		prefix = ""
+		param = ""
 	}
+	storage, prefix := parsePrefix(param)
 
 	var (
 		sources = []interface{}{}
@@ -110,7 +112,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	distribution := context.flags.Lookup("distribution").Value.String()
 
-	published, err := deb.NewPublishedRepo(prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -128,7 +130,13 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context.PublishedStorage(), context.CollectionFactory(), signer, context.Progress())
+	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -146,8 +154,13 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		prefix += "/"
 	}
 
-	context.Progress().Printf("\n%s been successfully published.\nPlease setup your webserver to serve directory '%s' with autoindexing.\n",
-		message, context.PublishedStorage().PublicPath())
+	context.Progress().Printf("\n%s been successfully published.\n", message)
+
+	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.LocalPublishedStorage); ok {
+		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
+			localStorage.PublicPath())
+	}
+
 	context.Progress().Printf("Now you can add following line to apt sources:\n")
 	context.Progress().Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
 	if utils.StrSliceHasItem(published.Architectures, "source") {
@@ -162,7 +175,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 func makeCmdPublishSnapshot() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSnapshotOrRepo,
-		UsageLine: "snapshot <name> [<prefix>]",
+		UsageLine: "snapshot <name> [[<endpoint>:]<prefix>]",
 		Short:     "publish snapshot",
 		Long: `
 Command publishes snapshot as Debian repository ready to be consumed
@@ -186,9 +199,12 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd
 }

cmd/publish_switch.go

@@ -19,7 +19,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	var (
 		names    []string
@@ -27,15 +27,17 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	)
 
 	if len(args) == len(components)+2 {
-		prefix = args[1]
+		param = args[1]
 		names = args[2:]
 	} else {
 		names = args[1:]
 	}
 
+	storage, prefix := parsePrefix(param)
+
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByPrefixDistribution(prefix, distribution)
+	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -77,7 +79,13 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context.PublishedStorage(), context.CollectionFactory(), signer, context.Progress())
+	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -88,7 +96,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	}
 
 	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		context.PublishedStorage(), context.CollectionFactory(), context.Progress())
+		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -101,7 +109,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 func makeCmdPublishSwitch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSwitch,
-		UsageLine: "switch <distribution> [<prefix>] <new-snapshot>",
+		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
 		Short:     "update published repository by switching to new snapshot",
 		Long: `
 Command switches in-place published repository with new snapshot contents. All
@@ -123,8 +131,11 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd
 }

cmd/publish_update.go

@@ -15,15 +15,16 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	if len(args) == 2 {
-		prefix = args[1]
+		param = args[1]
 	}
+	storage, prefix := parsePrefix(param)
 
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByPrefixDistribution(prefix, distribution)
+	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -47,7 +48,13 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context.PublishedStorage(), context.CollectionFactory(), signer, context.Progress())
+	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -58,7 +65,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		context.PublishedStorage(), context.CollectionFactory(), context.Progress())
+		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -71,7 +78,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 func makeCmdPublishUpdate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishUpdate,
-		UsageLine: "update <distribution> [<prefix>]",
+		UsageLine: "update <distribution> [[<endpoint>:]<prefix>]",
 		Short:     "update published local repository",
 		Long: `
 Command re-publishes (updates) published local repository. <distribution>
@@ -91,7 +98,10 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd
 }

cmd/repo.go

@@ -19,6 +19,8 @@ func makeCmdRepo() *commander.Command {
 			makeCmdRepoMove(),
 			makeCmdRepoRemove(),
 			makeCmdRepoShow(),
+			makeCmdRepoRename(),
+			makeCmdRepoSearch(),
 		},
 	}
 }

cmd/repo_add.go

@@ -40,6 +40,8 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
+	forceReplace := context.flags.Lookup("force-replace").Value.Get().(bool)
+
 	packageFiles := []string{}
 	failedFiles := []string{}
 
@@ -59,14 +61,16 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 					return nil
 				}
 
-				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
+				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+					strings.HasSuffix(info.Name(), ".dsc") {
 					packageFiles = append(packageFiles, path)
 				}
 
 				return nil
 			})
 		} else {
-			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
+			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+				strings.HasSuffix(info.Name(), ".dsc") {
 				packageFiles = append(packageFiles, location)
 			} else {
 				context.Progress().ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
@@ -79,6 +83,10 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 	processedFiles := []string{}
 	sort.Strings(packageFiles)
 
+	if forceReplace {
+		list.PrepareIndex()
+	}
+
 	for _, file := range packageFiles {
 		var (
 			stanza deb.Stanza
@@ -87,6 +95,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 		candidateProcessedFiles := []string{}
 		isSourcePackage := strings.HasSuffix(file, ".dsc")
+		isUdebPackage := strings.HasSuffix(file, ".udeb")
 
 		if isSourcePackage {
 			stanza, err = deb.GetControlFileFromDsc(file, verifier)
@@ -99,7 +108,11 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 			}
 		} else {
 			stanza, err = deb.GetControlFileFromDeb(file)
-			p = deb.NewPackageFromControlFile(stanza)
+			if isUdebPackage {
+				p = deb.NewUdebPackageFromControlFile(stanza)
+			} else {
+				p = deb.NewPackageFromControlFile(stanza)
+			}
 		}
 		if err != nil {
 			context.Progress().ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
@@ -155,6 +168,14 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 			continue
 		}
 
+		if forceReplace {
+			conflictingPackages := list.Search(deb.Dependency{Pkg: p.Name, Version: p.Version, Architecture: p.Architecture}, true)
+			for _, cp := range conflictingPackages {
+				context.Progress().ColoredPrintf("@r[-]@| %s removed due to conflict with package being added", cp)
+				list.Remove(cp)
+			}
+		}
+
 		err = list.Add(p)
 		if err != nil {
 			context.Progress().ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
@@ -202,8 +223,8 @@ func makeCmdRepoAdd() *commander.Command {
 		UsageLine: "add <name> <package file.deb>|<directory> ...",
 		Short:     "add packages to local repository",
 		Long: `
-Command adds packages to local repository from .deb (binary packages) and .dsc (source packages) files.
-When importing from directory aptly would do recursive scan looking for all files matching *.deb or *.dsc
+Command adds packages to local repository from .deb, .udeb (binary packages) and .dsc (source packages) files.
+When importing from directory aptly would do recursive scan looking for all files matching *.[u]deb or *.dsc
 patterns. Every file discovered would be analyzed to extract metadata, package would then be created and added
 to the database. Files would be imported to internal package pool. For source packages, all required files are
 added automatically as well. Extra files for source package should be in the same directory as *.dsc file.
@@ -216,6 +237,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("remove-files", false, "remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
 
 	return cmd
 }

cmd/repo_copy.go

@@ -8,10 +8,10 @@ import (
 func makeCmdRepoCopy() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "copy <src-name> <dst-name> <package-spec> ...",
+		UsageLine: "copy <src-name> <dst-name> <package-query> ...",
 		Short:     "copy packages between local repositories",
 		Long: `
-Command copy copies packages matching <package-spec> from local repo
+Command copy copies packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
 Example:

cmd/repo_edit.go

@@ -50,7 +50,7 @@ func makeCmdRepoEdit() *commander.Command {
 		UsageLine: "edit <name>",
 		Short:     "edit properties of local repository",
 		Long: `
-Command edit allows to change metadata of local repository:
+Command edit allows one to change metadata of local repository:
 comment, default distribution and component.
 
 Example:

cmd/repo_import.go

@@ -8,10 +8,10 @@ import (
 func makeCmdRepoImport() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "import <src-mirror> <dst-repo> <package-spec> ...",
+		UsageLine: "import <src-mirror> <dst-repo> <package-query> ...",
 		Short:     "import packages from mirror to local repository",
 		Long: `
-Command import looks up packages matching <package-spec> in mirror <src-mirror>
+Command import looks up packages matching <package-query> in mirror <src-mirror>
 and copies them to local repo <dst-repo>.
 
 Example:

cmd/repo_move.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"sort"
@@ -105,7 +106,15 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	toProcess, err := srcList.Filter(args[2:], withDeps, dstList, context.DependencyOptions(), architecturesList)
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i := 0; i < len(args)-2; i++ {
+		queries[i], err = query.Parse(args[i+2])
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+	}
+
+	toProcess, err := srcList.Filter(queries, withDeps, dstList, context.DependencyOptions(), architecturesList)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
@@ -162,10 +171,10 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 func makeCmdRepoMove() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "move <src-name> <dst-name> <package-spec> ...",
+		UsageLine: "move <src-name> <dst-name> <package-query> ...",
 		Short:     "move packages between local repositories",
 		Long: `
-Command move moves packages matching <package-spec> from local repo
+Command move moves packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
 Example:

cmd/repo_remove.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -33,8 +34,16 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
+	queries := make([]deb.PackageQuery, len(args)-1)
+	for i := 0; i < len(args)-1; i++ {
+		queries[i], err = query.Parse(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to remove: %s", err)
+		}
+	}
+
 	list.PrepareIndex()
-	toRemove, err := list.Filter(args[1:], false, nil, 0, nil)
+	toRemove, err := list.Filter(queries, false, nil, 0, nil)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -62,10 +71,10 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 func makeCmdRepoRemove() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoRemove,
-		UsageLine: "remove <name> <package-spec> ...",
+		UsageLine: "remove <name> <package-query> ...",
 		Short:     "remove packages from local repository",
 		Long: `
-Commands removes packages matching <package-spec> from local repository
+Commands removes packages matching <package-query> from local repository
 <name>. If removed packages are not referenced by other repos or
 snapshots, they can be removed completely (including files) by running
 'aptly db cleanup'.

cmd/repo_rename.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyRepoRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.LocalRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	repo, err = context.CollectionFactory().LocalRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().LocalRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: local repo %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nLocal repo %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdRepoRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames local repository",
+		Long: `
+Command changes name of the local repo. Local repo name should be unique.
+
+Example:
+
+  $ aptly repo rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/repo_search.go

@@ -0,0 +1,26 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdRepoSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> <package-query>",
+		Short:     "search repo for packages matching query",
+		Long: `
+Command search displays list of packages in local repository that match package query
+
+Example:
+
+    $ aptly repo search my-software '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+
+	return cmd
+}

cmd/run.go

@@ -0,0 +1,44 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/commander"
+)
+
+// Run runs single command starting from root cmd with args, optionally initializing context
+func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode int) {
+	defer func() {
+		if r := recover(); r != nil {
+			fatal, ok := r.(*FatalError)
+			if !ok {
+				panic(r)
+			}
+			fmt.Println("ERROR:", fatal.Message)
+			returnCode = fatal.ReturnCode
+		}
+	}()
+
+	returnCode = 0
+
+	flags, args, err := cmd.ParseFlags(cmdArgs)
+	if err != nil {
+		Fatal(err)
+	}
+
+	if initContext {
+		err = InitContext(flags)
+		if err != nil {
+			Fatal(err)
+		}
+		defer ShutdownContext()
+	}
+
+	context.UpdateFlags(flags)
+
+	err = cmd.Dispatch(args)
+	if err != nil {
+		Fatal(err)
+	}
+
+	return
+}

cmd/serve.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
@@ -83,7 +84,7 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	publicPath := context.PublishedStorage().PublicPath()
+	publicPath := context.GetPublishedStorage("").(aptly.LocalPublishedStorage).PublicPath()
 	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)

cmd/snapshot.go

@@ -17,6 +17,9 @@ func makeCmdSnapshot() *commander.Command {
 			makeCmdSnapshotDiff(),
 			makeCmdSnapshotMerge(),
 			makeCmdSnapshotDrop(),
+			makeCmdSnapshotRename(),
+			makeCmdSnapshotSearch(),
+			makeCmdSnapshotFilter(),
 		},
 	}
 }

cmd/snapshot_create.go

@@ -23,6 +23,11 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
+		err = repo.CheckLock()
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
 		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)

cmd/snapshot_filter.go

@@ -0,0 +1,107 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"sort"
+	"strings"
+)
+
+func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 3 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+
+	// Load <source> snapshot
+	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Convert snapshot to package list
+	context.Progress().Printf("Loading packages (%d)...\n", source.RefList().Len())
+	packageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	context.Progress().Printf("Building indexes...\n")
+	packageList.PrepareIndex()
+
+	// Calculate architectures
+	var architecturesList []string
+
+	if len(context.ArchitecturesList()) > 0 {
+		architecturesList = context.ArchitecturesList()
+	} else {
+		architecturesList = packageList.Architectures(false)
+	}
+
+	sort.Strings(architecturesList)
+
+	if len(architecturesList) == 0 && withDeps {
+		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+	}
+
+	// Initial queries out of arguments
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i, arg := range args[2:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+	}
+
+	// Filter with dependencies as requested
+	result, err := packageList.Filter(queries, withDeps, nil, context.DependencyOptions(), architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Create <destination> snapshot
+	destination := deb.NewSnapshotFromPackageList(args[1], []*deb.Snapshot{source}, result,
+		fmt.Sprintf("Filtered '%s', query was: '%s'", source.Name, strings.Join(args[2:], " ")))
+
+	err = context.CollectionFactory().SnapshotCollection().Add(destination)
+	if err != nil {
+		return fmt.Errorf("unable to create snapshot: %s", err)
+	}
+
+	context.Progress().Printf("\nSnapshot %s successfully filtered.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+
+	return err
+}
+
+func makeCmdSnapshotFilter() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotFilter,
+		UsageLine: "filter <source> <destination> <package-query> ...",
+		Short:     "filter packages in snapshot producing another snapshot",
+		Long: `
+Command filter does filtering in snapshot <source>, producing another
+snapshot <destination>. Packages could be specified simply
+as 'package-name' or as package queries.
+
+Example:
+
+    $ aptly snapshot filter wheezy-main wheezy-required 'Priorioty (required)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-filter", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependent packages as well")
+
+	return cmd
+}

cmd/snapshot_list.go

@@ -7,6 +7,46 @@ import (
 	"sort"
 )
 
+// Snapshot sorting methods
+const (
+	SortName = iota
+	SortTime
+)
+
+type snapshotListToSort struct {
+	list       []*deb.Snapshot
+	sortMethod int
+}
+
+func parseSortMethod(sortMethod string) (int, error) {
+	switch sortMethod {
+	case "time", "Time":
+		return SortTime, nil
+	case "name", "Name":
+		return SortName, nil
+	}
+
+	return -1, fmt.Errorf("sorting method \"%s\" unknown", sortMethod)
+}
+
+func (s snapshotListToSort) Swap(i, j int) {
+	s.list[i], s.list[j] = s.list[j], s.list[i]
+}
+
+func (s snapshotListToSort) Less(i, j int) bool {
+	switch s.sortMethod {
+	case SortName:
+		return s.list[i].Name < s.list[j].Name
+	case SortTime:
+		return s.list[i].CreatedAt.Before(s.list[j].CreatedAt)
+	}
+	panic("unknown sort method")
+}
+
+func (s snapshotListToSort) Len() int {
+	return len(s.list)
+}
+
 func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
@@ -15,32 +55,35 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	}
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
 
-	snapshots := make([]string, context.CollectionFactory().SnapshotCollection().Len())
+	snapshotsToSort := &snapshotListToSort{}
+	snapshotsToSort.list = make([]*deb.Snapshot, context.CollectionFactory().SnapshotCollection().Len())
+	snapshotsToSort.sortMethod, err = parseSortMethod(sortMethodString)
+	if err != nil {
+		return err
+	}
 
 	i := 0
 	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		if raw {
-			snapshots[i] = snapshot.Name
-		} else {
-			snapshots[i] = snapshot.String()
-		}
+		snapshotsToSort.list[i] = snapshot
 		i++
+
 		return nil
 	})
 
-	sort.Strings(snapshots)
+	sort.Sort(snapshotsToSort)
 
 	if raw {
-		for _, snapshot := range snapshots {
-			fmt.Printf("%s\n", snapshot)
+		for _, snapshot := range snapshotsToSort.list {
+			fmt.Printf("%s\n", snapshot.Name)
 		}
 	} else {
-		if len(snapshots) > 0 {
+		if len(snapshotsToSort.list) > 0 {
 			fmt.Printf("List of snapshots:\n")
 
-			for _, snapshot := range snapshots {
-				fmt.Printf(" * %s\n", snapshot)
+			for _, snapshot := range snapshotsToSort.list {
+				fmt.Printf(" * %s\n", snapshot.String())
 			}
 
 			fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
@@ -67,6 +110,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+	cmd.Flag.String("sort", "name", "display list in 'name' or creation 'time' order")
 
 	return cmd
 }

cmd/snapshot_pull.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"sort"
@@ -18,6 +19,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 
 	noDeps := context.flags.Lookup("no-deps").Value.Get().(bool)
 	noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
+	allMatches := context.flags.Lookup("all-matches").Value.Get().(bool)
 
 	// Load <name> snapshot
 	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
@@ -75,77 +77,57 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
 	}
 
-	// Initial dependencies out of arguments
-	initialDependencies := make([]deb.Dependency, len(args)-3)
-	for i, arg := range args[3:] {
-		initialDependencies[i], err = deb.ParseDependency(arg)
-		if err != nil {
-			return fmt.Errorf("unable to parse argument: %s", err)
-		}
+	// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
+	var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
+	for _, arch := range architecturesList {
+		archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
 	}
 
-	// Perform pull
-	for _, arch := range architecturesList {
-		dependencies := make([]deb.Dependency, len(initialDependencies), 2*len(initialDependencies))
-		for i := range dependencies {
-			dependencies[i] = initialDependencies[i]
-			dependencies[i].Architecture = arch
+	// Initial queries out of arguments
+	queries := make([]deb.PackageQuery, len(args)-3)
+	for i, arg := range args[3:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+		// Add architecture filter
+		queries[i] = &deb.AndQuery{queries[i], archQuery}
+	}
+
+	// Filter with dependencies as requested
+	result, err := sourcePackageList.Filter(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+	result.PrepareIndex()
+
+	alreadySeen := map[string]bool{}
+
+	result.ForEachIndexed(func(pkg *deb.Package) error {
+		key := pkg.Architecture + "_" + pkg.Name
+		_, seen := alreadySeen[key]
+
+		// If we haven't seen such name-architecture pair and were instructed to remove, remove it
+		if !noRemove && !seen {
+			// Remove all packages with the same name and architecture
+			pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
+			for _, p := range pS {
+				packageList.Remove(p)
+				context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
+			}
 		}
 
-		// Go over list of initial dependencies + list of dependencies found
-		for i := 0; i < len(dependencies); i++ {
-			dep := dependencies[i]
-
-			// Search for package that can satisfy dependencies
-			pkg := sourcePackageList.Search(dep)
-			if pkg == nil {
-				context.Progress().ColoredPrintf("@y[!]@| @!Dependency %s can't be satisfied with source %s@|", &dep, source)
-				continue
-			}
-
-			if !noRemove {
-				// Remove all packages with the same name and architecture
-				for p := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}); p != nil; {
-					packageList.Remove(p)
-					context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
-					p = packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name})
-				}
-			}
-
-			// Add new discovered package
+		// If !allMatches, add only first matching name-arch package
+		if !seen || allMatches {
 			packageList.Add(pkg)
 			context.Progress().ColoredPrintf("@g[+]@| %s added", pkg)
-
-			if noDeps {
-				continue
-			}
-
-			// Find missing dependencies for single added package
-			pL := deb.NewPackageList()
-			pL.Add(pkg)
-
-			var missing []deb.Dependency
-			missing, err = pL.VerifyDependencies(context.DependencyOptions(), []string{arch}, packageList, nil)
-			if err != nil {
-				context.Progress().ColoredPrintf("@y[!]@| @!Error while verifying dependencies for pkg %s: %s@|", pkg, err)
-			}
-
-			// Append missing dependencies to the list of dependencies to satisfy
-			for _, misDep := range missing {
-				found := false
-				for _, d := range dependencies {
-					if d == misDep {
-						found = true
-						break
-					}
-				}
-
-				if !found {
-					dependencies = append(dependencies, misDep)
-				}
-			}
 		}
-	}
+
+		alreadySeen[key] = true
+
+		return nil
+	})
+	alreadySeen = nil
 
 	if context.flags.Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nNot creating snapshot, as dry run was requested.\n")
@@ -167,14 +149,14 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 func makeCmdSnapshotPull() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotPull,
-		UsageLine: "pull <name> <source> <destination> <package-name> ...",
+		UsageLine: "pull <name> <source> <destination> <package-query> ...",
 		Short:     "pull packages from another snapshot",
 		Long: `
 Command pull pulls new packages along with its' dependencies to snapshot <name>
 from snapshot <source>. Pull can upgrade package version in <name> with
 versions from <source> following dependencies. New snapshot <destination>
 is created as a result of this process. Packages could be specified simply
-as 'package-name' or as dependency 'package-name (>= version)'.
+as 'package-name' or as package queries.
 
 Example:
 
@@ -186,6 +168,7 @@ Example:
 	cmd.Flag.Bool("dry-run", false, "don't create destination snapshot, just show what would be pulled")
 	cmd.Flag.Bool("no-deps", false, "don't process dependencies, just pull listed packages")
 	cmd.Flag.Bool("no-remove", false, "don't remove other package versions when pulling package")
+	cmd.Flag.Bool("all-matches", false, "pull all the packages that satisfy the dependency version requirements")
 
 	return cmd
 }

cmd/snapshot_rename.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlySnapshotRename(cmd *commander.Command, args []string) error {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().SnapshotCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: snapshot %s already exists", newName)
+	}
+
+	snapshot.Name = newName
+	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nSnapshot %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdSnapshotRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames snapshot",
+		Long: `
+Command changes name of the snapshot. Snapshot name should be unique.
+
+Example:
+
+  $ aptly snapshot rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/snapshot_search.go

@@ -0,0 +1,125 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"sort"
+)
+
+func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+	command := cmd.Parent.Name()
+
+	var reflist *deb.PackageRefList
+
+	if command == "snapshot" {
+		snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = snapshot.RefList()
+	} else if command == "mirror" {
+		repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else if command == "repo" {
+		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else {
+		panic("unknown command")
+	}
+
+	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	list.PrepareIndex()
+
+	q, err := query.Parse(args[1])
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+	architecturesList := []string{}
+
+	if withDeps {
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
+		} else {
+			architecturesList = list.Architectures(false)
+		}
+
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+		}
+	}
+
+	result, err := list.Filter([]deb.PackageQuery{q}, withDeps,
+		nil, context.DependencyOptions(), architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	result.ForEach(func(p *deb.Package) error {
+		context.Progress().Printf("%s\n", p)
+		return nil
+	})
+
+	return err
+}
+
+func makeCmdSnapshotSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> <package-query>",
+		Short:     "search snapshot for packages matching query",
+		Long: `
+Command search displays list of packages in snapshot that match package query
+
+Example:
+
+    $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-search", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+
+	return cmd
+}

cmd/task.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdTask() *commander.Command {
+	return &commander.Command{
+		UsageLine: "task",
+		Short:     "manage aptly tasks",
+		Subcommands: []*commander.Command{
+			makeCmdTaskRun(),
+		},
+	}
+}

cmd/task_run.go

@@ -0,0 +1,148 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mattn/go-shellwords"
+	"github.com/smira/commander"
+)
+
+func aptlyTaskRun(cmd *commander.Command, args []string) error {
+	var err error
+	var cmdList [][]string
+
+	if filename := cmd.Flag.Lookup("filename").Value.Get().(string); filename != "" {
+		var text string
+		cmdArgs := []string{}
+
+		if finfo, err := os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
+			return fmt.Errorf("no such file, %s\n", filename)
+		}
+
+		fmt.Println("Reading file...\n")
+
+		file, err := os.Open(filename)
+
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		scanner := bufio.NewScanner(file)
+
+		for scanner.Scan() {
+			text = strings.TrimSpace(scanner.Text()) + ","
+			parsedArgs, _ := shellwords.Parse(text)
+			cmdArgs = append(cmdArgs, parsedArgs...)
+		}
+
+		if err = scanner.Err(); err != nil {
+			return err
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("the file is empty")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else if len(args) == 0 {
+		var text string
+		cmdArgs := []string{}
+
+		fmt.Println("Please enter one command per line and leave one blank when finished.")
+
+		reader := bufio.NewReader(os.Stdin)
+		for {
+			fmt.Printf("> ")
+			text, _ = reader.ReadString('\n')
+			if text == "\n" {
+				break
+			} else {
+				text = strings.TrimSpace(text) + ","
+				parsedArgs, _ := shellwords.Parse(text)
+				cmdArgs = append(cmdArgs, parsedArgs...)
+			}
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("nothing entered")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else {
+		cmdList = formatCommands(args)
+	}
+
+	commandErrored := false
+
+	for i, command := range cmdList {
+		if !commandErrored {
+			context.Progress().ColoredPrintf("@g%d) [Running]: %s@!", (i + 1), strings.Join(command, " "))
+			context.Progress().ColoredPrintf("\n@yBegin command output: ----------------------------@!")
+			context.Progress().Flush()
+
+			returnCode := Run(RootCommand(), command, false)
+			if returnCode != 0 {
+				commandErrored = true
+			}
+			context.Progress().ColoredPrintf("\n@yEnd command output: ------------------------------@!")
+			CleanupContext()
+		} else {
+			context.Progress().ColoredPrintf("@r%d) [Skipping]: %s@!", (i + 1), strings.Join(command, " "))
+		}
+	}
+
+	if commandErrored {
+		err = fmt.Errorf("at least one command has reported an error")
+	}
+
+	return err
+}
+
+func formatCommands(args []string) [][]string {
+	var cmd []string
+	var cmdArray [][]string
+
+	for _, s := range args {
+		if sTrimmed := strings.TrimRight(s, ","); sTrimmed != s {
+			cmd = append(cmd, sTrimmed)
+			cmdArray = append(cmdArray, cmd)
+			cmd = []string{}
+		} else {
+			cmd = append(cmd, s)
+		}
+	}
+
+	if len(cmd) > 0 {
+		cmdArray = append(cmdArray, cmd)
+	}
+
+	return cmdArray
+}
+
+func makeCmdTaskRun() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyTaskRun,
+		UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
+		Short:     "run aptly tasks",
+		Long: `
+Command helps origanise multiple aptly commands in one single aptly task, running as single thread.
+
+Example:
+
+  $ aptly task run
+  > repo create local
+  > repo add local pkg1
+  > publish repo local
+  > serve
+  >
+
+`,
+	}
+
+	cmd.Flag.String("filename", "", "specifies the filename that contains the commands to run")
+	return cmd
+}

database/leveldb.go

@@ -24,12 +24,14 @@ type Storage interface {
 	KeysByPrefix(prefix []byte) [][]byte
 	FetchByPrefix(prefix []byte) [][]byte
 	Close() error
+	ReOpen() error
 	StartBatch()
 	FinishBatch() error
 	CompactDB() error
 }
 
 type levelDB struct {
+	path  string
 	db    *leveldb.DB
 	batch *leveldb.Batch
 }
@@ -39,17 +41,21 @@ var (
 	_ Storage = &levelDB{}
 )
 
-// OpenDB opens (creates) LevelDB database
-func OpenDB(path string) (Storage, error) {
+func internalOpen(path string) (*leveldb.DB, error) {
 	o := &opt.Options{
 		Filter: filter.NewBloomFilter(10),
 	}
 
-	db, err := leveldb.OpenFile(path, o)
+	return leveldb.OpenFile(path, o)
+}
+
+// OpenDB opens (creates) LevelDB database
+func OpenDB(path string) (Storage, error) {
+	db, err := internalOpen(path)
 	if err != nil {
 		return nil, err
 	}
-	return &levelDB{db: db}, nil
+	return &levelDB{db: db, path: path}, nil
 }
 
 // RecoverDB recovers LevelDB database from corruption
@@ -147,7 +153,23 @@ func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
 
 // Close finishes DB work
 func (l *levelDB) Close() error {
-	return l.db.Close()
+	if l.db == nil {
+		return nil
+	}
+	err := l.db.Close()
+	l.db = nil
+	return err
+}
+
+// Reopen tries to re-open the database
+func (l *levelDB) ReOpen() error {
+	if l.db != nil {
+		return nil
+	}
+
+	var err error
+	l.db, err = internalOpen(l.path)
+	return err
 }
 
 // StartBatch starts batch processing of keys

database/leveldb_test.go

@@ -155,3 +155,23 @@ func (s *LevelDBSuite) TestCompactDB(c *C) {
 
 	c.Check(s.db.CompactDB(), IsNil)
 }
+
+func (s *LevelDBSuite) TestReOpen(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	err = s.db.Close()
+	c.Assert(err, IsNil)
+
+	err = s.db.ReOpen()
+	c.Assert(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}

deb/format.go

@@ -12,7 +12,8 @@ type Stanza map[string]string
 
 // Canonical order of fields in stanza
 var canocialOrder = []string{"Package", "Origin", "Label", "Suite", "Version", "Installed-Size", "Priority", "Section", "Maintainer",
-	"Architecture", "Codename", "Date", "Architectures", "Components", "Description", "MD5sum", "MD5Sum", "SHA1", "SHA256"}
+	"Architecture", "Codename", "Date", "Architectures", "Components", "Description", "MD5sum", "MD5Sum", "SHA1", "SHA256",
+	"Archive", "Component"}
 
 // Copy returns copy of Stanza
 func (s Stanza) Copy() (result Stanza) {

deb/index_files.go

@@ -0,0 +1,254 @@
+package deb
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type indexFiles struct {
+	publishedStorage aptly.PublishedStorage
+	basePath         string
+	renameMap        map[string]string
+	generatedFiles   map[string]utils.ChecksumInfo
+	tempDir          string
+	suffix           string
+	indexes          map[string]*indexFile
+}
+
+type indexFile struct {
+	parent       *indexFiles
+	discardable  bool
+	compressable bool
+	signable     bool
+	relativePath string
+	tempFilename string
+	tempFile     *os.File
+	w            *bufio.Writer
+}
+
+func (file *indexFile) BufWriter() (*bufio.Writer, error) {
+	if file.w == nil {
+		var err error
+		file.tempFilename = filepath.Join(file.parent.tempDir, strings.Replace(file.relativePath, "/", "_", -1))
+		file.tempFile, err = os.Create(file.tempFilename)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create temporary index file: %s", err)
+		}
+
+		file.w = bufio.NewWriter(file.tempFile)
+	}
+
+	return file.w, nil
+}
+
+func (file *indexFile) Finalize(signer utils.Signer) error {
+	if file.w == nil {
+		if file.discardable {
+			return nil
+		}
+		file.BufWriter()
+	}
+
+	err := file.w.Flush()
+	if err != nil {
+		file.tempFile.Close()
+		return fmt.Errorf("unable to write to index file: %s", err)
+	}
+
+	if file.compressable {
+		err = utils.CompressFile(file.tempFile)
+		if err != nil {
+			file.tempFile.Close()
+			return fmt.Errorf("unable to compress index file: %s", err)
+		}
+	}
+
+	file.tempFile.Close()
+
+	exts := []string{""}
+	if file.compressable {
+		exts = append(exts, ".gz", ".bz2")
+	}
+
+	for _, ext := range exts {
+		var checksumInfo utils.ChecksumInfo
+
+		checksumInfo, err = utils.ChecksumsForFile(file.tempFilename + ext)
+		if err != nil {
+			return fmt.Errorf("unable to collect checksums: %s", err)
+		}
+		file.parent.generatedFiles[file.relativePath+ext] = checksumInfo
+	}
+
+	err = file.parent.publishedStorage.MkDir(filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)))
+	if err != nil {
+		return fmt.Errorf("unable to create dir: %s", err)
+	}
+
+	for _, ext := range exts {
+		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext),
+			file.tempFilename+ext)
+		if err != nil {
+			return fmt.Errorf("unable to publish file: %s", err)
+		}
+
+		if file.parent.suffix != "" {
+			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] =
+				filepath.Join(file.parent.basePath, file.relativePath+ext)
+		}
+	}
+
+	if file.signable && signer != nil {
+		err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
+		if err != nil {
+			return fmt.Errorf("unable to detached sign file: %s", err)
+		}
+
+		err = signer.ClearSign(file.tempFilename, filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
+		if err != nil {
+			return fmt.Errorf("unable to clearsign file: %s", err)
+		}
+
+		if file.parent.suffix != "" {
+			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
+				filepath.Join(file.parent.basePath, file.relativePath+".gpg")
+			file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] =
+				filepath.Join(file.parent.basePath, "In"+file.relativePath)
+		}
+
+		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
+			file.tempFilename+".gpg")
+		if err != nil {
+			return fmt.Errorf("unable to publish file: %s", err)
+		}
+
+		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix),
+			filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
+		if err != nil {
+			return fmt.Errorf("unable to publish file: %s", err)
+		}
+	}
+
+	return nil
+}
+
+func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string) *indexFiles {
+	return &indexFiles{
+		publishedStorage: publishedStorage,
+		basePath:         basePath,
+		renameMap:        make(map[string]string),
+		generatedFiles:   make(map[string]utils.ChecksumInfo),
+		tempDir:          tempDir,
+		suffix:           suffix,
+		indexes:          make(map[string]*indexFile),
+	}
+}
+
+func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
+	key := fmt.Sprintf("pi-%s-%s-%s", component, arch, udeb)
+	file, ok := files.indexes[key]
+	if !ok {
+		var relativePath string
+
+		if arch == "source" {
+			relativePath = filepath.Join(component, "source", "Sources")
+		} else {
+			if udeb {
+				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
+			} else {
+				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
+			}
+		}
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  false,
+			compressable: true,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
+func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexFile {
+	key := fmt.Sprintf("ri-%s-%s-%s", component, arch, udeb)
+	file, ok := files.indexes[key]
+	if !ok {
+		var relativePath string
+
+		if arch == "source" {
+			relativePath = filepath.Join(component, "source", "Release")
+		} else {
+			if udeb {
+				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Release")
+			} else {
+				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Release")
+			}
+		}
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  udeb,
+			compressable: false,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
+func (files *indexFiles) ReleaseFile() *indexFile {
+	return &indexFile{
+		parent:       files,
+		discardable:  false,
+		compressable: false,
+		signable:     true,
+		relativePath: "Release",
+	}
+}
+
+func (files *indexFiles) FinalizeAll(progress aptly.Progress) (err error) {
+	if progress != nil {
+		progress.InitBar(int64(len(files.indexes)), false)
+		defer progress.ShutdownBar()
+	}
+
+	for _, file := range files.indexes {
+		err = file.Finalize(nil)
+		if err != nil {
+			return
+		}
+		if progress != nil {
+			progress.AddBar(1)
+		}
+	}
+
+	files.indexes = make(map[string]*indexFile)
+
+	return
+}
+
+func (files *indexFiles) RenameFiles() error {
+	var err error
+
+	for oldName, newName := range files.renameMap {
+		err = files.publishedStorage.RenameFile(oldName, newName)
+		if err != nil {
+			return fmt.Errorf("unable to rename: %s", err)
+		}
+	}
+
+	return nil
+}

deb/list.go

@@ -5,7 +5,6 @@ import (
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
 	"sort"
-	"strings"
 )
 
 // Dependency options
@@ -42,6 +41,7 @@ type PackageList struct {
 // Verify interface
 var (
 	_ sort.Interface = &PackageList{}
+	_ PackageCatalog = &PackageList{}
 )
 
 // NewPackageList creates empty package list
@@ -101,7 +101,7 @@ func (l *PackageList) Add(p *Package) error {
 			l.providesIndex[provides] = append(l.providesIndex[provides], p)
 		}
 
-		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= p.Name })
+		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.lessPackages(p, l.packagesIndex[j]) })
 
 		// insert p into l.packagesIndex in position i
 		l.packagesIndex = append(l.packagesIndex, nil)
@@ -123,6 +123,22 @@ func (l *PackageList) ForEach(handler func(*Package) error) error {
 	return err
 }
 
+// ForEachIndexed calls handler for each package in list in indexed order
+func (l *PackageList) ForEachIndexed(handler func(*Package) error) error {
+	if !l.indexed {
+		panic("list not indexed, can't iterate")
+	}
+
+	var err error
+	for _, p := range l.packagesIndex {
+		err = handler(p)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
 // Len returns number of packages in the list
 func (l *PackageList) Len() int {
 	return len(l.packages)
@@ -220,6 +236,7 @@ func depSliceDeduplicate(s []Dependency) []Dependency {
 //
 // Analysis would be peformed for each architecture, in specified sources
 func (l *PackageList) VerifyDependencies(options int, architectures []string, sources *PackageList, progress aptly.Progress) ([]Dependency, error) {
+	l.PrepareIndex()
 	missing := make([]Dependency, 0, 128)
 
 	if progress != nil {
@@ -229,7 +246,7 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 	for _, arch := range architectures {
 		cache := make(map[string]bool, 2048)
 
-		for _, p := range l.packages {
+		for _, p := range l.packagesIndex {
 			if progress != nil {
 				progress.AddBar(1)
 			}
@@ -247,7 +264,6 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 				variants = depSliceDeduplicate(variants)
 
 				variantsMissing := make([]Dependency, 0, len(variants))
-				missingCount := 0
 
 				for _, dep := range variants {
 					if dep.Architecture == "" {
@@ -255,35 +271,23 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 					}
 
 					hash := dep.Hash()
-					r, ok := cache[hash]
-					if ok {
-						if !r {
-							missingCount++
-						}
-						continue
+					satisfied, ok := cache[hash]
+					if !ok {
+						satisfied = sources.Search(dep, false) != nil
+						cache[hash] = satisfied
 					}
 
-					if sources.Search(dep) == nil {
+					if !satisfied && !ok {
 						variantsMissing = append(variantsMissing, dep)
-						missingCount++
-					} else {
-						cache[hash] = true
+					}
+
+					if satisfied && options&DepFollowAllVariants == 0 {
+						variantsMissing = nil
+						break
 					}
 				}
 
-				if options&DepFollowAllVariants == DepFollowAllVariants {
-					missing = append(missing, variantsMissing...)
-					for _, dep := range variantsMissing {
-						cache[dep.Hash()] = false
-					}
-				} else {
-					if missingCount == len(variants) {
-						missing = append(missing, variantsMissing...)
-						for _, dep := range variantsMissing {
-							cache[dep.Hash()] = false
-						}
-					}
-				}
+				missing = append(missing, variantsMissing...)
 			}
 		}
 	}
@@ -300,13 +304,29 @@ func (l *PackageList) Swap(i, j int) {
 	l.packagesIndex[i], l.packagesIndex[j] = l.packagesIndex[j], l.packagesIndex[i]
 }
 
-// Compare compares two names in lexographical order
+func (l *PackageList) lessPackages(iPkg, jPkg *Package) bool {
+	if iPkg.Name == jPkg.Name {
+		cmp := CompareVersions(iPkg.Version, jPkg.Version)
+		if cmp == 0 {
+			return iPkg.Architecture < jPkg.Architecture
+		}
+		return cmp == 1
+	}
+
+	return iPkg.Name < jPkg.Name
+}
+
+// Less compares two packages by name (lexographical) and version (latest to oldest)
 func (l *PackageList) Less(i, j int) bool {
-	return l.packagesIndex[i].Name < l.packagesIndex[j].Name
+	return l.lessPackages(l.packagesIndex[i], l.packagesIndex[j])
 }
 
 // PrepareIndex prepares list for indexing
 func (l *PackageList) PrepareIndex() {
+	if l.indexed {
+		return
+	}
+
 	l.packagesIndex = make([]*Package, l.Len())
 	l.providesIndex = make(map[string][]*Package, 128)
 
@@ -325,16 +345,49 @@ func (l *PackageList) PrepareIndex() {
 	l.indexed = true
 }
 
-// Search searches package index for specified package
-func (l *PackageList) Search(dep Dependency) *Package {
+// Scan searches package index using full scan
+func (l *PackageList) Scan(q PackageQuery) (result *PackageList) {
+	result = NewPackageList()
+	for _, pkg := range l.packages {
+		if q.Matches(pkg) {
+			result.Add(pkg)
+		}
+	}
+
+	return
+}
+
+// SearchSupported returns true for PackageList
+func (l *PackageList) SearchSupported() bool {
+	return true
+}
+
+// SearchByKey looks up package by exact key reference
+func (l *PackageList) SearchByKey(arch, name, version string) (result *PackageList) {
+	result = NewPackageList()
+
+	pkg := l.packages["P"+arch+" "+name+" "+version]
+	if pkg != nil {
+		result.Add(pkg)
+	}
+
+	return
+}
+
+// Search searches package index for specified package(s) using optimized queries
+func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*Package) {
 	if !l.indexed {
 		panic("list not indexed, can't search")
 	}
 
 	if dep.Relation == VersionDontCare {
 		for _, p := range l.providesIndex[dep.Pkg] {
-			if p.MatchesArchitecture(dep.Architecture) {
-				return p
+			if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
+				searchResults = append(searchResults, p)
+
+				if !allMatches {
+					break
+				}
 			}
 		}
 	}
@@ -344,16 +397,21 @@ func (l *PackageList) Search(dep Dependency) *Package {
 	for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
 		p := l.packagesIndex[i]
 		if p.MatchesDependency(dep) {
-			return p
+			searchResults = append(searchResults, p)
+
+			if !allMatches {
+				break
+			}
 		}
 
 		i++
 	}
-	return nil
+
+	return
 }
 
 // Filter filters package index by specified queries (ORed together), possibly pulling dependencies
-func (l *PackageList) Filter(queries []string, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
+func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
 	if !l.indexed {
 		panic("list not indexed, can't filter")
 	}
@@ -361,53 +419,17 @@ func (l *PackageList) Filter(queries []string, withDependencies bool, source *Pa
 	result := NewPackageList()
 
 	for _, query := range queries {
-		isDepQuery := strings.IndexAny(query, " (){}=<>") != -1
-
-		if !isDepQuery {
-			// try to interpret query as package string representation
-
-			// convert Package.String() to Package.Key()
-			i := strings.Index(query, "_")
-			if i != -1 {
-				pkg, query := query[:i], query[i+1:]
-				j := strings.LastIndex(query, "_")
-				if j != -1 {
-					version, arch := query[:j], query[j+1:]
-					p := l.packages["P"+arch+" "+pkg+" "+version]
-					if p != nil {
-						result.Add(p)
-						continue
-					}
-				}
-			}
-		}
-
-		// try as dependency
-		dep, err := ParseDependency(query)
-		if err != nil {
-			if isDepQuery {
-				return nil, err
-			}
-			// parsing failed, but probably that wasn't a dep query
-			continue
-		}
-
-		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= dep.Pkg })
-
-		for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
-			p := l.packagesIndex[i]
-			if p.MatchesDependency(dep) {
-				result.Add(p)
-			}
-			i++
-		}
+		result.Append(query.Query(l))
 	}
 
 	if withDependencies {
 		added := result.Len()
+		result.PrepareIndex()
 
 		dependencySource := NewPackageList()
-		dependencySource.Append(source)
+		if source != nil {
+			dependencySource.Append(source)
+		}
 		dependencySource.Append(result)
 		dependencySource.PrepareIndex()
 
@@ -423,11 +445,22 @@ func (l *PackageList) Filter(queries []string, withDependencies bool, source *Pa
 
 			// try to satisfy dependencies
 			for _, dep := range missing {
-				p := l.Search(dep)
-				if p != nil {
-					result.Add(p)
-					dependencySource.Add(p)
-					added++
+				// dependency might have already been satisfied
+				// with packages already been added
+				if result.Search(dep, false) != nil {
+					continue
+				}
+
+				searchResults := l.Search(dep, false)
+				if searchResults != nil {
+					for _, p := range searchResults {
+						result.Add(p)
+						dependencySource.Add(p)
+						added++
+						if dependencyOptions&DepFollowAllVariants == 0 {
+							break
+						}
+					}
 				}
 			}
 		}

deb/list_test.go

@@ -3,10 +3,47 @@ package deb
 import (
 	"errors"
 	. "launchpad.net/gocheck"
+	"regexp"
 	"sort"
 	"strings"
 )
 
+type containsChecker struct {
+	*CheckerInfo
+}
+
+func (c *containsChecker) Check(params []interface{}, names []string) (result bool, error string) {
+	var (
+		pkgSlice1 []*Package
+		pkgSlice2 []*Package
+		ok        bool
+	)
+
+	pkgMap := make(map[*Package]bool)
+
+	pkgSlice1, ok = params[0].([]*Package)
+	if !ok {
+		return false, "The first parameter is not a Package slice"
+	}
+	pkgSlice2, ok = params[1].([]*Package)
+	if !ok {
+		return false, "The second parameter is not a Package slice"
+	}
+
+	for _, pkg := range pkgSlice2 {
+		pkgMap[pkg] = true
+	}
+
+	for _, pkg := range pkgSlice1 {
+		if _, ok := pkgMap[pkg]; !ok {
+			return false, ""
+		}
+	}
+	return true, ""
+}
+
+var Contains = &containsChecker{&CheckerInfo{Name: "Contains", Params: []string{"Container", "Expected to contain"}}}
+
 type PackageListSuite struct {
 	// Simple list with "real" packages from stanzas
 	list                   *PackageList
@@ -14,8 +51,10 @@ type PackageListSuite struct {
 
 	// Mocked packages in list
 	packages       []*Package
+	packages2      []*Package
 	sourcePackages []*Package
 	il             *PackageList
+	il2            *PackageList
 }
 
 var _ = Suite(&PackageListSuite{})
@@ -60,6 +99,20 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 	}
 	s.il.PrepareIndex()
 
+	s.il2 = NewPackageList()
+	s.packages2 = []*Package{
+		&Package{Name: "mailer", Version: "3.5.8", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		&Package{Name: "sendmail", Version: "1.0", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		&Package{Name: "app", Version: "1.1-bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		&Package{Name: "app", Version: "1.1-bp2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		&Package{Name: "app", Version: "1.2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
+		&Package{Name: "app", Version: "3.0", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+	}
+	for _, p := range s.packages2 {
+		s.il2.Add(p)
+	}
+	s.il2.PrepareIndex()
+
 	s.sourcePackages = []*Package{
 		&Package{Name: "postfix", Version: "1.3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
 		&Package{Name: "app", Version: "1.1~bp1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
@@ -196,35 +249,60 @@ func (s *PackageListSuite) TestAppend(c *C) {
 }
 
 func (s *PackageListSuite) TestSearch(c *C) {
-	c.Check(func() { s.list.Search(Dependency{Architecture: "i386", Pkg: "app"}) }, Panics, "list not indexed, can't search")
+	//allMatches = False
+	c.Check(func() { s.list.Search(Dependency{Architecture: "i386", Pkg: "app"}, false) }, Panics, "list not indexed, can't search")
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "mail-agent"}), Equals, s.packages[4])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "puppy"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "mail-agent"}, false), DeepEquals, []*Package{s.packages[4]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "puppy"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp2"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp2"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1~~"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1~~"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~bp1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~~"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~~"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.0"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.2"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.0"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.2"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}, false), IsNil)
+
+	// search w/o version should return package with latest version
+	c.Check(s.il.Search(Dependency{Architecture: "source", Pkg: "dpkg"}, false), DeepEquals, []*Package{s.packages[13]})
+
+	// allMatches = True
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "mail-agent"}, true), Contains, []*Package{s.packages2[0], s.packages2[1]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "puppy"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "3"}, true), Contains, []*Package{s.packages2[5]})
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLess, Version: "1.2"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLess, Version: "1.1~"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.2"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1-bp1"}, true), Contains, []*Package{s.packages2[2]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.0"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreater, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreater, Version: "5.0"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}, true), Contains, []*Package{s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "5.0"}, true), IsNil)
 }
 
 func (s *PackageListSuite) TestFilter(c *C) {
-	c.Check(func() { s.list.Filter([]string{"abcd_0.3_i386"}, false, nil, 0, nil) }, Panics, "list not indexed, can't filter")
-
-	_, err := s.il.Filter([]string{"app >3)"}, false, nil, 0, nil)
-	c.Check(err, ErrorMatches, "unable to parse dependency.*")
+	c.Check(func() { s.list.Filter([]PackageQuery{&PkgQuery{"abcd", "0.3", "i386"}}, false, nil, 0, nil) }, Panics, "list not indexed, can't filter")
 
 	plString := func(l *PackageList) string {
 		list := make([]string, 0, l.Len())
@@ -237,25 +315,69 @@ func (s *PackageListSuite) TestFilter(c *C) {
 		return strings.Join(list, " ")
 	}
 
-	result, err := s.il.Filter([]string{"app_1.1~bp1_i386"}, false, nil, 0, nil)
+	result, err := s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
 
-	result, err = s.il.Filter([]string{"app_1.1~bp1_i386", "dpkg_1.7_source", "dpkg_1.8_amd64"}, false, nil, 0, nil)
+	result, err = s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, &PkgQuery{"dpkg", "1.7", "source"},
+		&PkgQuery{"dpkg", "1.8", "amd64"}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_source")
 
-	result, err = s.il.Filter([]string{"app", "dpkg (>>1.6.1-3)", "app (>=1.0)", "xyz", "aa (>>3.0)"}, false, nil, 0, nil)
+	result, err = s.il.Filter([]PackageQuery{
+		&DependencyQuery{Dep: Dependency{Pkg: "app"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "xyz"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "aa", Relation: VersionGreater, Version: "3.0"}}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source")
 
-	result, err = s.il.Filter([]string{"app {i386}"}, true, NewPackageList(), 0, []string{"i386"})
+	result, err = s.il.Filter([]PackageQuery{&DependencyQuery{Dep: Dependency{Pkg: "app", Architecture: "i386"}}}, true, NewPackageList(), 0, []string{"i386"})
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386")
 
-	result, err = s.il.Filter([]string{"app (>=0.9)", "lib", "data"}, true, NewPackageList(), 0, []string{"i386", "amd64"})
+	result, err = s.il.Filter([]PackageQuery{
+		&DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "0.9"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "lib"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "data"}}}, true, NewPackageList(), 0, []string{"i386", "amd64"})
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "")
+
+	result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_i386 data_1.1~bp1_all")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{&FieldQuery{Field: "Version", Relation: VersionGreaterOrEqual, Value: "1.0"},
+		&FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 data_1.1~bp1_all")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&NotQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.6.1-3_arm dpkg_1.6.1-3_source dpkg_1.7_source libx_1.5_arm")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionRegexp, Value: "i.*6", Regexp: regexp.MustCompile("i.*6")}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
 }
 
 func (s *PackageListSuite) TestVerifyDependencies(c *C) {

deb/package.go

@@ -24,6 +24,8 @@ type Package struct {
 	Provides []string
 	// Is this source package
 	IsSource bool
+	// Is this udeb package
+	IsUdeb bool
 	// Hash of files section
 	FilesHash uint64
 	// Is this >= 0.6 package?
@@ -43,7 +45,7 @@ func NewPackageFromControlFile(input Stanza) *Package {
 		Version:      input["Version"],
 		Architecture: input["Architecture"],
 		Source:       input["Source"],
-		V06Plus: 	  true,
+		V06Plus:      true,
 	}
 
 	delete(input, "Package")
@@ -92,7 +94,7 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 		Version:            input["Version"],
 		Architecture:       "source",
 		SourceArchitecture: input["Architecture"],
-		V06Plus: 	 	    true,
+		V06Plus:            true,
 	}
 
 	delete(input, "Package")
@@ -169,6 +171,14 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 	return result, nil
 }
 
+// NewUdebPackageFromControlFile creates .udeb Package from parsed Debian control file
+func NewUdebPackageFromControlFile(input Stanza) *Package {
+	p := NewPackageFromControlFile(input)
+	p.IsUdeb = true
+
+	return p
+}
+
 // Key returns unique key identifying package
 func (p *Package) Key(prefix string) []byte {
 	if p.V06Plus {
@@ -188,6 +198,73 @@ func (p *Package) String() string {
 	return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.Architecture)
 }
 
+// GetField returns fields from package
+func (p *Package) GetField(name string) string {
+	switch name {
+	// $Version is handled in FieldQuery
+	case "$Source":
+		if p.IsSource {
+			return ""
+		}
+		source := p.Source
+		if source == "" {
+			return p.Name
+		} else if pos := strings.Index(source, "("); pos != -1 {
+			return strings.TrimSpace(source[:pos])
+		}
+		return source
+	case "$SourceVersion":
+		if p.IsSource {
+			return ""
+		}
+		source := p.Source
+		if pos := strings.Index(source, "("); pos != -1 {
+			if pos2 := strings.LastIndex(source, ")"); pos2 != -1 && pos2 > pos {
+				return strings.TrimSpace(source[pos+1 : pos2])
+			}
+		}
+		return p.Version
+	case "$Architecture":
+		return p.Architecture
+	case "$PackageType":
+		if p.IsSource {
+			return "source"
+		}
+		if p.IsUdeb {
+			return "udeb"
+		}
+		return "deb"
+	case "Name":
+		return p.Name
+	case "Version":
+		return p.Version
+	case "Architecture":
+		if p.IsSource {
+			return p.SourceArchitecture
+		}
+		return p.Architecture
+	case "Source":
+		return p.Source
+	case "Depends":
+		return strings.Join(p.Deps().Depends, ", ")
+	case "Pre-Depends":
+		return strings.Join(p.Deps().PreDepends, ", ")
+	case "Suggests":
+		return strings.Join(p.Deps().Suggests, ", ")
+	case "Recommends":
+		return strings.Join(p.Deps().Recommends, ", ")
+	case "Provides":
+		return strings.Join(p.Provides, ", ")
+	case "Build-Depends":
+		return strings.Join(p.Deps().BuildDepends, ", ")
+	case "Build-Depends-Indep":
+		return strings.Join(p.Deps().BuildDependsInDep, ", ")
+	default:
+		return p.Extra()[name]
+	}
+	return ""
+}
+
 // MatchesArchitecture checks whether packages matches specified architecture
 func (p *Package) MatchesArchitecture(arch string) bool {
 	if p.Architecture == "all" && arch != "source" {
@@ -199,19 +276,23 @@ func (p *Package) MatchesArchitecture(arch string) bool {
 
 // MatchesDependency checks whether package matches specified dependency
 func (p *Package) MatchesDependency(dep Dependency) bool {
-	if dep.Pkg != p.Name {
-		return false
-	}
-
 	if dep.Architecture != "" && !p.MatchesArchitecture(dep.Architecture) {
 		return false
 	}
 
 	if dep.Relation == VersionDontCare {
-		return true
+		if utils.StrSliceHasItem(p.Provides, dep.Pkg) {
+			return true
+		}
+		return dep.Pkg == p.Name
+	}
+
+	if dep.Pkg != p.Name {
+		return false
 	}
 
 	r := CompareVersions(p.Version, dep.Version)
+
 	switch dep.Relation {
 	case VersionEqual:
 		return r == 0
@@ -223,6 +304,11 @@ func (p *Package) MatchesDependency(dep Dependency) bool {
 		return r <= 0
 	case VersionGreaterOrEqual:
 		return r >= 0
+	case VersionPatternMatch:
+		matched, err := filepath.Match(dep.Version, p.Version)
+		return err == nil && matched
+	case VersionRegexp:
+		return dep.Regexp.FindStringIndex(p.Version) != nil
 	}
 
 	panic("unknown relation")
@@ -389,7 +475,8 @@ func (p *Package) Equals(p2 *Package) bool {
 }
 
 // LinkFromPool links package file from pool to dist's pool location
-func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool, prefix string, component string) error {
+func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool,
+	prefix, component string, force bool) error {
 	poolDir, err := p.PoolDirectory()
 	if err != nil {
 		return err
@@ -404,7 +491,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 		relPath := filepath.Join("pool", component, poolDir)
 		publishedDirectory := filepath.Join(prefix, relPath)
 
-		err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePath)
+		err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePath, f.Checksums.MD5, force)
 		if err != nil {
 			return err
 		}

deb/package_collection.go

@@ -12,12 +12,19 @@ import (
 type PackageCollection struct {
 	db           database.Storage
 	encodeBuffer bytes.Buffer
+	codecHandle  *codec.MsgpackHandle
 }
 
+// Verify interface
+var (
+	_ PackageCatalog = &PackageCollection{}
+)
+
 // NewPackageCollection creates new PackageCollection and binds it to database
 func NewPackageCollection(db database.Storage) *PackageCollection {
 	return &PackageCollection{
-		db: db,
+		db:          db,
+		codecHandle: &codec.MsgpackHandle{},
 	}
 }
 
@@ -53,7 +60,7 @@ func (collection *PackageCollection) ByKey(key []byte) (*Package, error) {
 	if len(encoded) > 2 && (encoded[0] != 0xc1 || encoded[1] != 0x1) {
 		oldp := &oldPackage{}
 
-		decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+		decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 		err = decoder.Decode(oldp)
 		if err != nil {
 			return nil, err
@@ -88,7 +95,7 @@ func (collection *PackageCollection) ByKey(key []byte) (*Package, error) {
 			return nil, err
 		}
 	} else {
-		decoder := codec.NewDecoderBytes(encoded[2:], &codec.MsgpackHandle{})
+		decoder := codec.NewDecoderBytes(encoded[2:], collection.codecHandle)
 		err = decoder.Decode(p)
 		if err != nil {
 			return nil, err
@@ -109,7 +116,7 @@ func (collection *PackageCollection) loadExtra(p *Package) *Stanza {
 
 	stanza := &Stanza{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(stanza)
 	if err != nil {
 		panic("unable to decode extra")
@@ -127,7 +134,7 @@ func (collection *PackageCollection) loadDependencies(p *Package) *PackageDepend
 
 	deps := &PackageDependencies{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(deps)
 	if err != nil {
 		panic("unable to decode deps")
@@ -145,7 +152,7 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 
 	files := &PackageFiles{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(files)
 	if err != nil {
 		panic("unable to decode files")
@@ -156,7 +163,7 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 
 // Update adds or updates information about package in DB checking for conficts first
 func (collection *PackageCollection) Update(p *Package) error {
-	encoder := codec.NewEncoder(&collection.encodeBuffer, &codec.MsgpackHandle{})
+	encoder := codec.NewEncoder(&collection.encodeBuffer, collection.codecHandle)
 
 	collection.encodeBuffer.Reset()
 	collection.encodeBuffer.WriteByte(0xc1)
@@ -235,3 +242,49 @@ func (collection *PackageCollection) DeleteByKey(key []byte) error {
 	}
 	return nil
 }
+
+// Scan does full scan on all the packages
+func (collection *PackageCollection) Scan(q PackageQuery) (result *PackageList) {
+	result = NewPackageList()
+
+	for _, key := range collection.db.KeysByPrefix([]byte("P")) {
+		pkg, err := collection.ByKey(key)
+		if err != nil {
+			panic(fmt.Sprintf("unable to load package: %s", err))
+		}
+
+		if q.Matches(pkg) {
+			result.Add(pkg)
+		}
+	}
+
+	return
+}
+
+// Search is not implemented
+func (collection *PackageCollection) Search(dep Dependency, allMatches bool) (searchResults []*Package) {
+	panic("Not implemented")
+}
+
+// SearchSupported returns false
+func (collection *PackageCollection) SearchSupported() bool {
+	return false
+}
+
+// SearchByKey finds package by exact key
+func (collection *PackageCollection) SearchByKey(arch, name, version string) (result *PackageList) {
+	result = NewPackageList()
+
+	for _, key := range collection.db.KeysByPrefix([]byte(fmt.Sprintf("P%s %s %s", arch, name, version))) {
+		pkg, err := collection.ByKey(key)
+		if err != nil {
+			panic(fmt.Sprintf("unable to load package: %s", err))
+		}
+
+		if pkg.Architecture == arch && pkg.Name == name && pkg.Version == version {
+			result.Add(pkg)
+		}
+	}
+
+	return
+}

deb/package_test.go

@@ -7,6 +7,7 @@ import (
 	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
+	"regexp"
 )
 
 type PackageSuite struct {
@@ -27,6 +28,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
 	c.Check(p.IsSource, Equals, false)
+	c.Check(p.IsUdeb, Equals, false)
 	c.Check(p.Name, Equals, "alien-arena-common")
 	c.Check(p.Version, Equals, "7.40-2")
 	c.Check(p.Architecture, Equals, "i386")
@@ -39,11 +41,27 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 	c.Check(p.deps.Depends, DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)"})
 }
 
+func (s *PackageSuite) TestNewUdebFromPara(c *C) {
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	p := NewUdebPackageFromControlFile(stanza)
+
+	c.Check(p.IsSource, Equals, false)
+	c.Check(p.IsUdeb, Equals, true)
+	c.Check(p.Name, Equals, "dmidecode-udeb")
+	c.Check(p.Version, Equals, "2.11-9")
+	c.Check(p.Architecture, Equals, "amd64")
+	c.Check(p.Provides, DeepEquals, []string(nil))
+	c.Check(p.Files(), HasLen, 1)
+	c.Check(p.Files()[0].Filename, Equals, "dmidecode-udeb_2.11-9_amd64.udeb")
+	c.Check(p.deps.Depends, DeepEquals, []string{"libc6-udeb (>= 2.13)"})
+}
+
 func (s *PackageSuite) TestNewSourceFromPara(c *C) {
 	p, err := NewSourcePackageFromControlFile(s.sourceStanza)
 
 	c.Check(err, IsNil)
 	c.Check(p.IsSource, Equals, true)
+	c.Check(p.IsUdeb, Equals, false)
 	c.Check(p.Name, Equals, "access-modifier-checker")
 	c.Check(p.Version, Equals, "1.0-4")
 	c.Check(p.Architecture, Equals, "source")
@@ -90,16 +108,16 @@ func (s *PackageSuite) TestKey(c *C) {
 	c.Check(p.Key(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2 c8901eedd79ac51b"))
 	c.Check(p.Key("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2 c8901eedd79ac51b"))
 
-    p.V06Plus = false
-    c.Check(p.Key(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2"))
-    c.Check(p.Key("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2"))
+	p.V06Plus = false
+	c.Check(p.Key(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2"))
+	c.Check(p.Key("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2"))
 }
 
 func (s *PackageSuite) TestShortKey(c *C) {
-    p := NewPackageFromControlFile(s.stanza)
+	p := NewPackageFromControlFile(s.stanza)
 
-    c.Check(p.ShortKey(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2"))
-    c.Check(p.ShortKey("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2"))
+	c.Check(p.ShortKey(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2"))
+	c.Check(p.ShortKey("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2"))
 }
 
 func (s *PackageSuite) TestStanza(c *C) {
@@ -119,6 +137,65 @@ func (s *PackageSuite) TestString(c *C) {
 	c.Assert(p.String(), Equals, "alien-arena-common_7.40-2_i386")
 }
 
+func (s *PackageSuite) TestGetField(c *C) {
+	p := NewPackageFromControlFile(s.stanza.Copy())
+
+	stanza2 := s.stanza.Copy()
+	delete(stanza2, "Source")
+	stanza2["Provides"] = "app, game"
+	p2 := NewPackageFromControlFile(stanza2)
+
+	stanza3 := s.stanza.Copy()
+	stanza3["Source"] = "alien-arena (3.5)"
+	p3 := NewPackageFromControlFile(stanza3)
+
+	p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
+
+	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	p5 := NewUdebPackageFromControlFile(stanza5)
+
+	c.Check(p.GetField("$Source"), Equals, "alien-arena")
+	c.Check(p2.GetField("$Source"), Equals, "alien-arena-common")
+	c.Check(p3.GetField("$Source"), Equals, "alien-arena")
+	c.Check(p4.GetField("$Source"), Equals, "")
+	c.Check(p5.GetField("$Source"), Equals, "dmidecode")
+
+	c.Check(p.GetField("$SourceVersion"), Equals, "7.40-2")
+	c.Check(p2.GetField("$SourceVersion"), Equals, "7.40-2")
+	c.Check(p3.GetField("$SourceVersion"), Equals, "3.5")
+	c.Check(p4.GetField("$SourceVersion"), Equals, "")
+	c.Check(p5.GetField("$SourceVersion"), Equals, "2.11-9")
+
+	c.Check(p.GetField("$Architecture"), Equals, "i386")
+	c.Check(p4.GetField("$Architecture"), Equals, "source")
+	c.Check(p5.GetField("$Architecture"), Equals, "amd64")
+
+	c.Check(p.GetField("$PackageType"), Equals, "deb")
+	c.Check(p4.GetField("$PackageType"), Equals, "source")
+	c.Check(p5.GetField("$PackageType"), Equals, "udeb")
+
+	c.Check(p.GetField("Name"), Equals, "alien-arena-common")
+	c.Check(p4.GetField("Name"), Equals, "access-modifier-checker")
+
+	c.Check(p.GetField("Architecture"), Equals, "i386")
+	c.Check(p4.GetField("Architecture"), Equals, "all")
+
+	c.Check(p.GetField("Version"), Equals, "7.40-2")
+
+	c.Check(p.GetField("Source"), Equals, "alien-arena")
+	c.Check(p2.GetField("Source"), Equals, "")
+	c.Check(p3.GetField("Source"), Equals, "alien-arena (3.5)")
+	c.Check(p4.GetField("Source"), Equals, "")
+
+	c.Check(p.GetField("Depends"), Equals, "libc6 (>= 2.7), alien-arena-data (>= 7.40)")
+
+	c.Check(p.GetField("Provides"), Equals, "")
+	c.Check(p2.GetField("Provides"), Equals, "app, game")
+
+	c.Check(p.GetField("Section"), Equals, "contrib/games")
+	c.Check(p.GetField("Priority"), Equals, "extra")
+}
+
 func (s *PackageSuite) TestEquals(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
@@ -174,6 +251,9 @@ func (s *PackageSuite) TestMatchesDependency(c *C) {
 	// exact match
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionEqual, Version: "7.40-2"}), Equals, true)
 
+	// exact match, same version, no revision specified
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionEqual, Version: "7.40"}), Equals, false)
+
 	// different name
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena", Architecture: "i386", Relation: VersionEqual, Version: "7.40-2"}), Equals, false)
 
@@ -204,6 +284,29 @@ func (s *PackageSuite) TestMatchesDependency(c *C) {
 	// <=
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionLessOrEqual, Version: "7.40-2"}), Equals, true)
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionLessOrEqual, Version: "7.40-1"}), Equals, false)
+
+	// %
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-*"}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-[2]"}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-[2"}), Equals, false)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-[34]"}), Equals, false)
+
+	// ~
+	c.Check(
+		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
+			Regexp: regexp.MustCompile("7\\.40-.*")}), Equals, true)
+	c.Check(
+		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
+			Regexp: regexp.MustCompile("40")}), Equals, true)
+	c.Check(
+		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
+			Regexp: regexp.MustCompile("39-.*")}), Equals, false)
+
+	// Provides
+	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Relation: VersionDontCare}), Equals, false)
+	p.Provides = []string{"fun", "game"}
+	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Relation: VersionDontCare}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Architecture: "amd64", Relation: VersionDontCare}), Equals, false)
 }
 
 func (s *PackageSuite) TestGetDependencies(c *C) {
@@ -266,13 +369,13 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
 	c.Assert(err, IsNil)
 	file.Close()
 
-	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free")
+	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
 	c.Check(err, IsNil)
 	c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
 	c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena")
 
 	p.IsSource = true
-	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free")
+	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
 	c.Check(err, IsNil)
 	c.Check(p.Extra()["Directory"], Equals, "pool/non-free/a/alien-arena")
 }
@@ -376,3 +479,20 @@ Directory: pool/main/a/access-modifier-checker
 Priority: source
 Section: java
 `
+
+const udebPackageMeta = `Package: dmidecode-udeb
+Source: dmidecode
+Version: 2.11-9
+Installed-Size: 115
+Maintainer: Daniel Baumann <daniel.baumann@progress-technologies.net>
+Architecture: amd64
+Depends: libc6-udeb (>= 2.13)
+Description: SMBIOS/DMI table decoder (udeb)
+Description-md5: bdfb786c6a57097be8c8600b800e749f
+Section: debian-installer
+Priority: optional
+Filename: pool/main/d/dmidecode/dmidecode-udeb_2.11-9_amd64.udeb
+Size: 29188
+MD5sum: ae70341c4d96dcded89fa670bcfea31e
+SHA1: 9532ae4226a85805189a671ee0283f719d48a5ba
+SHA256: bbb3a2cb07f741c3995b6d4bb08d772d83582b93a0236d4ea7736bc0370fc320`

deb/publish.go

@@ -1,7 +1,6 @@
 package deb
 
 import (
-	"bufio"
 	"bytes"
 	"code.google.com/p/go-uuid/uuid"
 	"fmt"
@@ -9,6 +8,7 @@ import (
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/utils"
 	"github.com/ugorji/go/codec"
+	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -30,7 +30,8 @@ type repoSourceItem struct {
 type PublishedRepo struct {
 	// Internal unique ID
 	UUID string
-	// Prefix & distribution should be unique across all published repositories
+	// Storage & Prefix & distribution should be unique across all published repositories
+	Storage      string
 	Prefix       string
 	Distribution string
 	Origin       string
@@ -115,13 +116,15 @@ func walkUpTree(source interface{}, collectionFactory *CollectionFactory) (rootD
 
 // NewPublishedRepo creates new published repository
 //
+// storage is PublishedStorage name
 // prefix specifies publishing prefix
 // distribution and architectures are user-defined properties
 // components & sources are lists of component to source mapping (*Snapshot or *LocalRepo)
-func NewPublishedRepo(prefix string, distribution string, architectures []string,
+func NewPublishedRepo(storage, prefix, distribution string, architectures []string,
 	components []string, sources []interface{}, collectionFactory *CollectionFactory) (*PublishedRepo, error) {
 	result := &PublishedRepo{
 		UUID:          uuid.New(),
+		Storage:       storage,
 		Architectures: architectures,
 		Sources:       make(map[string]string),
 		sourceItems:   make(map[string]repoSourceItem),
@@ -165,6 +168,9 @@ func NewPublishedRepo(prefix string, distribution string, architectures []string
 		if distribution == "" || component == "" {
 			rootDistributions, rootComponents := walkUpTree(source, collectionFactory)
 			if distribution == "" {
+				for i := range rootDistributions {
+					rootDistributions[i] = strings.Replace(rootDistributions[i], "/", "-", -1)
+				}
 				discoveredDistributions = append(discoveredDistributions, rootDistributions...)
 			}
 			if component == "" {
@@ -224,6 +230,10 @@ func NewPublishedRepo(prefix string, distribution string, architectures []string
 		}
 	}
 
+	if strings.Index(distribution, "/") != -1 {
+		return nil, fmt.Errorf("invalid distribution %s, '/' is not allowed", distribution)
+	}
+
 	result.Distribution = distribution
 
 	return result, nil
@@ -265,13 +275,22 @@ func (p *PublishedRepo) String() string {
 		extra = " (" + extra + ")"
 	}
 
-	return fmt.Sprintf("%s/%s%s [%s] publishes %s", p.Prefix, p.Distribution, extra, strings.Join(p.Architectures, ", "),
+	return fmt.Sprintf("%s/%s%s [%s] publishes %s", p.StoragePrefix(), p.Distribution, extra, strings.Join(p.Architectures, ", "),
 		strings.Join(sources, ", "))
 }
 
+// StoragePrefix returns combined storage & prefix for the repo
+func (p *PublishedRepo) StoragePrefix() string {
+	result := p.Prefix
+	if p.Storage != "" {
+		result = p.Storage + ":" + p.Prefix
+	}
+	return result
+}
+
 // Key returns unique key identifying PublishedRepo
 func (p *PublishedRepo) Key() []byte {
-	return []byte("U" + p.Prefix + ">>" + p.Distribution)
+	return []byte("U" + p.StoragePrefix() + ">>" + p.Distribution)
 }
 
 // RefKey is a unique id for package reference list
@@ -379,8 +398,10 @@ func (p *PublishedRepo) GetLabel() string {
 }
 
 // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
-func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage aptly.PublishedStorage,
-	collectionFactory *CollectionFactory, signer utils.Signer, progress aptly.Progress) error {
+func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
+	collectionFactory *CollectionFactory, signer utils.Signer, progress aptly.Progress, forceOverwrite bool) error {
+	publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage)
+
 	err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
 	if err != nil {
 		return err
@@ -425,50 +446,55 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage
 		suffix = ".tmp"
 	}
 
-	generatedFiles := map[string]utils.ChecksumInfo{}
-	renameMap := map[string]string{}
-
 	if progress != nil {
 		progress.Printf("Generating metadata files and linking package files...\n")
 	}
 
+	var tempDir string
+	tempDir, err = ioutil.TempDir(os.TempDir(), "aptly")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempDir)
+
+	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix)
+
 	for component, list := range lists {
-		var relativePath string
+		hadUdebs := false
 
-		// For all architectures, generate packages/sources files
+		// For all architectures, pregenerate packages/sources files
 		for _, arch := range p.Architectures {
+			indexes.PackageIndex(component, arch, false)
+		}
+
+		if progress != nil {
+			progress.InitBar(int64(list.Len()), false)
+		}
+
+		err = list.ForEach(func(pkg *Package) error {
 			if progress != nil {
-				progress.InitBar(int64(list.Len()), false)
+				progress.AddBar(1)
 			}
 
-			if arch == "source" {
-				relativePath = filepath.Join(component, "source", "Sources")
-			} else {
-				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
-			}
-			err = publishedStorage.MkDir(filepath.Dir(filepath.Join(basePath, relativePath)))
-			if err != nil {
-				return err
-			}
-
-			var packagesFile *os.File
-			packagesFile, err = publishedStorage.CreateFile(filepath.Join(basePath, relativePath+suffix))
-			if err != nil {
-				return fmt.Errorf("unable to creates Packages file: %s", err)
-			}
-
-			if suffix != "" {
-				renameMap[filepath.Join(basePath, relativePath+suffix)] = filepath.Join(basePath, relativePath)
-			}
-
-			bufWriter := bufio.NewWriter(packagesFile)
-
-			err = list.ForEach(func(pkg *Package) error {
-				if progress != nil {
-					progress.AddBar(1)
-				}
+			matches := false
+			for _, arch := range p.Architectures {
 				if pkg.MatchesArchitecture(arch) {
-					err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, component)
+					matches = true
+					break
+				}
+			}
+
+			if matches {
+				hadUdebs = hadUdebs || pkg.IsUdeb
+				err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, component, forceOverwrite)
+				if err != nil {
+					return err
+				}
+			}
+
+			for _, arch := range p.Architectures {
+				if pkg.MatchesArchitecture(arch) {
+					bufWriter, err := indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
 					if err != nil {
 						return err
 					}
@@ -481,109 +507,63 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage
 					if err != nil {
 						return err
 					}
-
-					pkg.files = nil
-					pkg.deps = nil
-					pkg.extra = nil
-
 				}
-
-				return nil
-			})
-
-			if err != nil {
-				return fmt.Errorf("unable to process packages: %s", err)
 			}
 
-			err = bufWriter.Flush()
-			if err != nil {
-				return fmt.Errorf("unable to write Packages file: %s", err)
-			}
+			pkg.files = nil
+			pkg.deps = nil
+			pkg.extra = nil
 
-			err = utils.CompressFile(packagesFile)
-			if err != nil {
-				return fmt.Errorf("unable to compress Packages files: %s", err)
-			}
+			return nil
+		})
 
-			if suffix != "" {
-				renameMap[filepath.Join(basePath, relativePath+suffix+".gz")] = filepath.Join(basePath, relativePath+".gz")
-				renameMap[filepath.Join(basePath, relativePath+suffix+".bz2")] = filepath.Join(basePath, relativePath+".bz2")
-			}
+		if err != nil {
+			return fmt.Errorf("unable to process packages: %s", err)
+		}
 
-			packagesFile.Close()
+		if progress != nil {
+			progress.ShutdownBar()
+		}
 
-			var checksumInfo utils.ChecksumInfo
-			checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+suffix))
-			if err != nil {
-				return fmt.Errorf("unable to collect checksums: %s", err)
-			}
-			generatedFiles[relativePath] = checksumInfo
+		udebs := []bool{false}
+		if hadUdebs {
+			udebs = append(udebs, true)
 
-			checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+suffix+".gz"))
-			if err != nil {
-				return fmt.Errorf("unable to collect checksums: %s", err)
-			}
-			generatedFiles[relativePath+".gz"] = checksumInfo
-
-			checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+suffix+".bz2"))
-			if err != nil {
-				return fmt.Errorf("unable to collect checksums: %s", err)
-			}
-			generatedFiles[relativePath+".bz2"] = checksumInfo
-
-			if progress != nil {
-				progress.ShutdownBar()
+			// For all architectures, pregenerate .udeb indexes
+			for _, arch := range p.Architectures {
+				indexes.PackageIndex(component, arch, true)
 			}
 		}
 
 		// For all architectures, generate Release files
 		for _, arch := range p.Architectures {
-			release := make(Stanza)
-			release["Archive"] = p.Distribution
-			release["Architecture"] = arch
-			release["Component"] = component
-			release["Origin"] = p.GetOrigin()
-			release["Label"] = p.GetLabel()
+			for _, udeb := range udebs {
+				release := make(Stanza)
+				release["Archive"] = p.Distribution
+				release["Architecture"] = arch
+				release["Component"] = component
+				release["Origin"] = p.GetOrigin()
+				release["Label"] = p.GetLabel()
 
-			if arch == "source" {
-				relativePath = filepath.Join(component, "source", "Release")
-			} else {
-				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Release")
+				bufWriter, err := indexes.ReleaseIndex(component, arch, udeb).BufWriter()
+
+				err = release.WriteTo(bufWriter)
+				if err != nil {
+					return fmt.Errorf("unable to create Release file: %s", err)
+				}
 			}
-
-			var file *os.File
-			file, err = publishedStorage.CreateFile(filepath.Join(basePath, relativePath+suffix))
-			if err != nil {
-				return fmt.Errorf("unable to create Release file: %s", err)
-			}
-
-			if suffix != "" {
-				renameMap[filepath.Join(basePath, relativePath+suffix)] = filepath.Join(basePath, relativePath)
-			}
-
-			bufWriter := bufio.NewWriter(file)
-
-			err = release.WriteTo(bufWriter)
-			if err != nil {
-				return fmt.Errorf("unable to create Release file: %s", err)
-			}
-
-			err = bufWriter.Flush()
-			if err != nil {
-				return fmt.Errorf("unable to create Release file: %s", err)
-			}
-
-			file.Close()
-
-			var checksumInfo utils.ChecksumInfo
-			checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+suffix))
-			if err != nil {
-				return fmt.Errorf("unable to collect checksums: %s", err)
-			}
-			generatedFiles[relativePath] = checksumInfo
 		}
 	}
 
+	if progress != nil {
+		progress.Printf("Finalizing metadata files...\n")
+	}
+
+	err = indexes.FinalizeAll(progress)
+	if err != nil {
+		return err
+	}
+
 	release := make(Stanza)
 	release["Origin"] = p.GetOrigin()
 	release["Label"] = p.GetLabel()
@@ -597,64 +577,36 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage
 
 	release["Components"] = strings.Join(p.Components(), " ")
 
-	for path, info := range generatedFiles {
+	for path, info := range indexes.generatedFiles {
 		release["MD5Sum"] += fmt.Sprintf(" %s %8d %s\n", info.MD5, info.Size, path)
 		release["SHA1"] += fmt.Sprintf(" %s %8d %s\n", info.SHA1, info.Size, path)
 		release["SHA256"] += fmt.Sprintf(" %s %8d %s\n", info.SHA256, info.Size, path)
 	}
 
-	releaseFile, err := publishedStorage.CreateFile(filepath.Join(basePath, "Release"+suffix))
+	releaseFile := indexes.ReleaseFile()
+	bufWriter, err := releaseFile.BufWriter()
 	if err != nil {
-		return fmt.Errorf("unable to create Release file: %s", err)
+		return err
 	}
 
-	if suffix != "" {
-		renameMap[filepath.Join(basePath, "Release"+suffix)] = filepath.Join(basePath, "Release")
-	}
-
-	bufWriter := bufio.NewWriter(releaseFile)
-
 	err = release.WriteTo(bufWriter)
 	if err != nil {
 		return fmt.Errorf("unable to create Release file: %s", err)
 	}
 
-	err = bufWriter.Flush()
-	if err != nil {
-		return fmt.Errorf("unable to create Release file: %s", err)
-	}
-
-	releaseFilename := releaseFile.Name()
-	releaseFile.Close()
-
 	// Signing files might output to console, so flush progress writer first
 	if progress != nil {
 		progress.Flush()
 	}
 
-	if signer != nil {
-		err = signer.DetachedSign(releaseFilename, releaseFilename+".gpg")
-		if err != nil {
-			return fmt.Errorf("unable to sign Release file: %s", err)
-		}
-
-		err = signer.ClearSign(releaseFilename, filepath.Join(filepath.Dir(releaseFilename), "InRelease"+suffix))
-		if err != nil {
-			return fmt.Errorf("unable to sign Release file: %s", err)
-		}
-
-		if suffix != "" {
-			renameMap[filepath.Join(basePath, "Release"+suffix+".gpg")] = filepath.Join(basePath, "Release.gpg")
-			renameMap[filepath.Join(basePath, "InRelease"+suffix)] = filepath.Join(basePath, "InRelease")
-		}
-
+	err = releaseFile.Finalize(signer)
+	if err != nil {
+		return err
 	}
 
-	for oldName, newName := range renameMap {
-		err = publishedStorage.RenameFile(oldName, newName)
-		if err != nil {
-			return fmt.Errorf("unable to rename: %s", err)
-		}
+	err = indexes.RenameFiles()
+	if err != nil {
+		return err
 	}
 
 	return nil
@@ -663,8 +615,10 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage
 // RemoveFiles removes files that were created by Publish
 //
 // It can remove prefix fully, and part of pool (for specific component)
-func (p *PublishedRepo) RemoveFiles(publishedStorage aptly.PublishedStorage, removePrefix bool,
+func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStorageProvider, removePrefix bool,
 	removePoolComponents []string, progress aptly.Progress) error {
+	publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage)
+
 	// I. Easy: remove whole prefix (meta+packages)
 	if removePrefix {
 		err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists"), progress)
@@ -722,7 +676,7 @@ func NewPublishedRepoCollection(db database.Storage) *PublishedRepoCollection {
 // Add appends new repo to collection and saves it
 func (collection *PublishedRepoCollection) Add(repo *PublishedRepo) error {
 	if collection.CheckDuplicate(repo) != nil {
-		return fmt.Errorf("published repo with prefix/distribution %s/%s already exists", repo.Prefix, repo.Distribution)
+		return fmt.Errorf("published repo with storage/prefix/distribution %s/%s/%s already exists", repo.Storage, repo.Prefix, repo.Distribution)
 	}
 
 	err := collection.Update(repo)
@@ -737,7 +691,7 @@ func (collection *PublishedRepoCollection) Add(repo *PublishedRepo) error {
 // CheckDuplicate verifies that there's no published repo with the same name
 func (collection *PublishedRepoCollection) CheckDuplicate(repo *PublishedRepo) *PublishedRepo {
 	for _, r := range collection.list {
-		if r.Prefix == repo.Prefix && r.Distribution == repo.Distribution {
+		if r.Prefix == repo.Prefix && r.Distribution == repo.Distribution && r.Storage == repo.Storage {
 			return r
 		}
 	}
@@ -823,14 +777,17 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
 	return
 }
 
-// ByPrefixDistribution looks up repository by prefix & distribution
-func (collection *PublishedRepoCollection) ByPrefixDistribution(prefix, distribution string) (*PublishedRepo, error) {
+// ByStoragePrefixDistribution looks up repository by storage, prefix & distribution
+func (collection *PublishedRepoCollection) ByStoragePrefixDistribution(storage, prefix, distribution string) (*PublishedRepo, error) {
 	for _, r := range collection.list {
-		if r.Prefix == prefix && r.Distribution == distribution {
+		if r.Prefix == prefix && r.Distribution == distribution && r.Storage == storage {
 			return r, nil
 		}
 	}
-	return nil, fmt.Errorf("published repo with prefix/distribution %s/%s not found", prefix, distribution)
+	if storage != "" {
+		storage += ":"
+	}
+	return nil, fmt.Errorf("published repo with storage:prefix/distribution %s%s/%s not found", storage, prefix, distribution)
 }
 
 // ByUUID looks up repository by uuid
@@ -982,9 +939,9 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
 }
 
 // Remove removes published repository, cleaning up directories, files
-func (collection *PublishedRepoCollection) Remove(publishedStorage aptly.PublishedStorage, prefix, distribution string,
-	collectionFactory *CollectionFactory, progress aptly.Progress) error {
-	repo, err := collection.ByPrefixDistribution(prefix, distribution)
+func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly.PublishedStorageProvider,
+	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
+	repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return err
 	}
@@ -999,7 +956,7 @@ func (collection *PublishedRepoCollection) Remove(publishedStorage aptly.Publish
 			repoPosition = i
 			continue
 		}
-		if r.Prefix == repo.Prefix {
+		if r.Storage == repo.Storage && r.Prefix == repo.Prefix {
 			removePrefix = false
 
 			rComponents := r.Components()
@@ -1012,7 +969,7 @@ func (collection *PublishedRepoCollection) Remove(publishedStorage aptly.Publish
 		}
 	}
 
-	err = repo.RemoveFiles(publishedStorage, removePrefix, removePoolComponents, progress)
+	err = repo.RemoveFiles(publishedStorageProvider, removePrefix, removePoolComponents, progress)
 	if err != nil {
 		return err
 	}
@@ -1021,7 +978,8 @@ func (collection *PublishedRepoCollection) Remove(publishedStorage aptly.Publish
 		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
 	if len(cleanComponents) > 0 {
-		err = collection.CleanupPrefixComponentFiles(repo.Prefix, cleanComponents, publishedStorage, collectionFactory, progress)
+		err = collection.CleanupPrefixComponentFiles(repo.Prefix, cleanComponents,
+			publishedStorageProvider.GetPublishedStorage(storage), collectionFactory, progress)
 		if err != nil {
 			return err
 		}

deb/publish_test.go

@@ -3,10 +3,12 @@ package deb
 import (
 	"bytes"
 	"errors"
+	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/files"
 	"github.com/ugorji/go/codec"
+	"io/ioutil"
 	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
@@ -37,25 +39,41 @@ func (n *NullSigner) SetKey(keyRef string) {
 func (n *NullSigner) SetKeyRing(keyring, secretKeyring string) {
 }
 
+func (n *NullSigner) SetPassphrase(passphrase, passphraseFile string) {
+}
+
 func (n *NullSigner) DetachedSign(source string, destination string) error {
-	return nil
+	return ioutil.WriteFile(destination, []byte{}, 0644)
 }
 
 func (n *NullSigner) ClearSign(source string, destination string) error {
-	return nil
+	return ioutil.WriteFile(destination, []byte{}, 0644)
+}
+
+type FakeStorageProvider struct {
+	storages map[string]aptly.PublishedStorage
+}
+
+func (p *FakeStorageProvider) GetPublishedStorage(name string) aptly.PublishedStorage {
+	storage, ok := p.storages[name]
+	if !ok {
+		panic(fmt.Sprintf("unknown storage: %#v", name))
+	}
+	return storage
 }
 
 type PublishedRepoSuite struct {
 	PackageListMixinSuite
-	repo, repo2, repo3, repo4 *PublishedRepo
-	root                      string
-	publishedStorage          aptly.PublishedStorage
-	packagePool               aptly.PackagePool
-	localRepo                 *LocalRepo
-	snapshot, snapshot2       *Snapshot
-	db                        database.Storage
-	factory                   *CollectionFactory
-	packageCollection         *PackageCollection
+	repo, repo2, repo3, repo4, repo5    *PublishedRepo
+	root, root2                         string
+	provider                            *FakeStorageProvider
+	publishedStorage, publishedStorage2 *files.PublishedStorage
+	packagePool                         aptly.PackagePool
+	localRepo                           *LocalRepo
+	snapshot, snapshot2                 *Snapshot
+	db                                  database.Storage
+	factory                             *CollectionFactory
+	packageCollection                   *PackageCollection
 }
 
 var _ = Suite(&PublishedRepoSuite{})
@@ -68,9 +86,14 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.root = c.MkDir()
 	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.root2 = c.MkDir()
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{
+		"":            s.publishedStorage,
+		"files:other": s.publishedStorage2}}
 	s.packagePool = files.NewPackagePool(s.root)
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	repo.packageRefs = s.reflist
 	s.factory.RemoteRepoCollection().Add(repo)
 
@@ -89,13 +112,15 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.packageCollection.Update(s.p2)
 	s.packageCollection.Update(s.p3)
 
-	s.repo, _ = NewPublishedRepo("ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
 
-	s.repo2, _ = NewPublishedRepo("ppa", "maverick", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo2, _ = NewPublishedRepo("", "ppa", "maverick", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 
-	s.repo3, _ = NewPublishedRepo("linux", "natty", nil, []string{"main", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	s.repo3, _ = NewPublishedRepo("", "linux", "natty", nil, []string{"main", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
 
-	s.repo4, _ = NewPublishedRepo("ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo4, _ = NewPublishedRepo("", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 
 	poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
 	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
@@ -132,16 +157,19 @@ func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) {
 	c.Check(s.repo3.RefList("main").Len(), Equals, 3)
 	c.Check(s.repo3.RefList("contrib").Len(), Equals, 3)
 
-	c.Check(func() { NewPublishedRepo(".", "a", nil, nil, nil, s.factory) }, PanicMatches, "publish with empty sources")
+	c.Check(func() { NewPublishedRepo("", ".", "a", nil, nil, nil, s.factory) }, PanicMatches, "publish with empty sources")
 	c.Check(func() {
-		NewPublishedRepo(".", "a", nil, []string{"main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+		NewPublishedRepo("", ".", "a", nil, []string{"main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
 	}, PanicMatches, "sources and components should be equal in size")
 	c.Check(func() {
-		NewPublishedRepo(".", "a", nil, []string{"main", "contrib"}, []interface{}{s.localRepo, s.snapshot2}, s.factory)
+		NewPublishedRepo("", ".", "a", nil, []string{"main", "contrib"}, []interface{}{s.localRepo, s.snapshot2}, s.factory)
 	}, PanicMatches, "interface conversion:.*")
 
-	_, err := NewPublishedRepo(".", "a", nil, []string{"main", "main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	_, err := NewPublishedRepo("", ".", "a", nil, []string{"main", "main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
 	c.Check(err, ErrorMatches, "duplicate component name: main")
+
+	_, err = NewPublishedRepo("", ".", "wheezy/updates", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	c.Check(err, ErrorMatches, "invalid distribution wheezy/updates, '/' is not allowed")
 }
 
 func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
@@ -200,7 +228,7 @@ func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
 			errorExpected: "invalid prefix .*",
 		},
 	} {
-		repo, err := NewPublishedRepo(t.prefix, "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+		repo, err := NewPublishedRepo("", t.prefix, "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
 		if t.errorExpected != "" {
 			c.Check(err, ErrorMatches, t.errorExpected)
 		} else {
@@ -210,49 +238,56 @@ func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
 }
 
 func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
-	repo, err := NewPublishedRepo("ppa", "", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
+	repo, err := NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"main"})
 
-	repo, err = NewPublishedRepo("ppa", "wheezy", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "wheezy", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "wheezy")
 	c.Check(repo.Components(), DeepEquals, []string{"main"})
 
-	repo, err = NewPublishedRepo("ppa", "", nil, []string{"non-free"}, []interface{}{s.snapshot}, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"non-free"}, []interface{}{s.snapshot}, s.factory)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"non-free"})
 
-	repo, err = NewPublishedRepo("ppa", "squeeze", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "squeeze", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"main"})
 
-	repo, err = NewPublishedRepo("ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 	c.Check(err, ErrorMatches, "unable to guess distribution name, please specify explicitly")
 
 	s.localRepo.DefaultDistribution = "precise"
 	s.localRepo.DefaultComponent = "contrib"
 	s.factory.LocalRepoCollection().Update(s.localRepo)
 
-	repo, err = NewPublishedRepo("ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "precise")
 	c.Check(repo.Components(), DeepEquals, []string{"contrib"})
 
-	repo, err = NewPublishedRepo("ppa", "", nil, []string{"", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	s.localRepo.DefaultDistribution = "precise/updates"
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "precise-updates")
+	c.Check(repo.Components(), DeepEquals, []string{"contrib"})
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"contrib", "main"})
 
-	repo, err = NewPublishedRepo("ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
 	c.Check(err, ErrorMatches, "duplicate component name: main")
 }
 
 func (s *PublishedRepoSuite) TestPublish(c *C) {
-	err := s.repo.Publish(s.packagePool, s.publishedStorage, s.factory, &NullSigner{}, nil)
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false)
 	c.Assert(err, IsNil)
 
 	c.Check(s.repo.Architectures, DeepEquals, []string{"i386"})
@@ -299,7 +334,7 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 }
 
 func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
-	err := s.repo.Publish(s.packagePool, s.publishedStorage, s.factory, nil, nil)
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
 	c.Assert(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists)
@@ -307,7 +342,7 @@ func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
 }
 
 func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) {
-	err := s.repo2.Publish(s.packagePool, s.publishedStorage, s.factory, nil, nil)
+	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
 	c.Assert(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
@@ -315,22 +350,30 @@ func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) {
 }
 
 func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) {
-	err := s.repo4.Publish(s.packagePool, s.publishedStorage, s.factory, nil, nil)
+	err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
 	c.Assert(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/main/source/Release"), PathExists)
 }
 
+func (s *PublishedRepoSuite) TestPublishOtherStorage(c *C) {
+	err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	c.Assert(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), Not(PathExists))
+}
+
 func (s *PublishedRepoSuite) TestString(c *C) {
 	c.Check(s.repo.String(), Equals,
 		"ppa/squeeze [] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
 	c.Check(s.repo2.String(), Equals,
 		"ppa/maverick [] publishes {main: [local1]: comment1}")
-	repo, _ := NewPublishedRepo("", "squeeze", []string{"s390"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	repo, _ := NewPublishedRepo("", "", "squeeze", []string{"s390"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
 	c.Check(repo.String(), Equals,
 		"./squeeze [s390] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
-	repo, _ = NewPublishedRepo("", "squeeze", []string{"i386", "amd64"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	repo, _ = NewPublishedRepo("", "", "squeeze", []string{"i386", "amd64"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
 	c.Check(repo.String(), Equals,
 		"./squeeze [i386, amd64] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
 	repo.Origin = "myorigin"
@@ -341,10 +384,13 @@ func (s *PublishedRepoSuite) TestString(c *C) {
 		"./squeeze (origin: myorigin, label: mylabel) [i386, amd64] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
 	c.Check(s.repo3.String(), Equals,
 		"linux/natty [] publishes {contrib: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}, {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	c.Check(s.repo5.String(), Equals,
+		"files:other:ppa/maverick [source] publishes {main: [local1]: comment1}")
 }
 
 func (s *PublishedRepoSuite) TestKey(c *C) {
 	c.Check(s.repo.Key(), DeepEquals, []byte("Uppa>>squeeze"))
+	c.Check(s.repo5.Key(), DeepEquals, []byte("Ufiles:other:ppa>>maverick"))
 }
 
 func (s *PublishedRepoSuite) TestRefKey(c *C) {
@@ -372,13 +418,13 @@ func (s *PublishedRepoSuite) TestEncodeDecode(c *C) {
 
 type PublishedRepoCollectionSuite struct {
 	PackageListMixinSuite
-	db                         database.Storage
-	factory                    *CollectionFactory
-	snapshotCollection         *SnapshotCollection
-	collection                 *PublishedRepoCollection
-	snap1, snap2               *Snapshot
-	localRepo                  *LocalRepo
-	repo1, repo2, repo3, repo4 *PublishedRepo
+	db                                database.Storage
+	factory                           *CollectionFactory
+	snapshotCollection                *SnapshotCollection
+	collection                        *PublishedRepoCollection
+	snap1, snap2                      *Snapshot
+	localRepo                         *LocalRepo
+	repo1, repo2, repo3, repo4, repo5 *PublishedRepo
 }
 
 var _ = Suite(&PublishedRepoCollectionSuite{})
@@ -398,10 +444,11 @@ func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
 	s.localRepo = NewLocalRepo("local1", "comment1")
 	s.factory.LocalRepoCollection().Add(s.localRepo)
 
-	s.repo1, _ = NewPublishedRepo("ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
-	s.repo2, _ = NewPublishedRepo("", "anaconda", []string{}, []string{"main", "contrib"}, []interface{}{s.snap2, s.snap1}, s.factory)
-	s.repo3, _ = NewPublishedRepo("ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap2}, s.factory)
-	s.repo4, _ = NewPublishedRepo("ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main", "contrib"}, []interface{}{s.snap2, s.snap1}, s.factory)
+	s.repo3, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap2}, s.factory)
+	s.repo4, _ = NewPublishedRepo("", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 
 	s.collection = s.factory.PublishedRepoCollection()
 }
@@ -410,8 +457,8 @@ func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) {
 	s.db.Close()
 }
 
-func (s *PublishedRepoCollectionSuite) TestAddByPrefixDistribution(c *C) {
-	r, err := s.collection.ByPrefixDistribution("ppa", "anaconda")
+func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C) {
+	r, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	c.Assert(s.collection.Add(s.repo1), IsNil)
@@ -421,8 +468,9 @@ func (s *PublishedRepoCollectionSuite) TestAddByPrefixDistribution(c *C) {
 	c.Assert(s.collection.Add(s.repo3), ErrorMatches, ".*already exists")
 	c.Assert(s.collection.CheckDuplicate(s.repo3), Equals, s.repo1)
 	c.Assert(s.collection.Add(s.repo4), IsNil)
+	c.Assert(s.collection.Add(s.repo5), IsNil)
 
-	r, err = s.collection.ByPrefixDistribution("ppa", "anaconda")
+	r, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, IsNil)
 
 	err = s.collection.LoadComplete(r, s.factory)
@@ -430,12 +478,15 @@ func (s *PublishedRepoCollectionSuite) TestAddByPrefixDistribution(c *C) {
 	c.Assert(r.String(), Equals, s.repo1.String())
 
 	collection := NewPublishedRepoCollection(s.db)
-	r, err = collection.ByPrefixDistribution("ppa", "anaconda")
+	r, err = collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, IsNil)
 
 	err = s.collection.LoadComplete(r, s.factory)
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, s.repo1.String())
+
+	r, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "precise")
+	c.Check(r.String(), Equals, s.repo5.String())
 }
 
 func (s *PublishedRepoCollectionSuite) TestByUUID(c *C) {
@@ -457,14 +508,14 @@ func (s *PublishedRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 	c.Assert(s.collection.Update(s.repo4), IsNil)
 
 	collection := NewPublishedRepoCollection(s.db)
-	r, err := collection.ByPrefixDistribution("ppa", "anaconda")
+	r, err := collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, IsNil)
 	c.Assert(r.sourceItems["main"].snapshot, IsNil)
 	c.Assert(s.collection.LoadComplete(r, s.factory), IsNil)
 	c.Assert(r.Sources["main"], Equals, s.repo1.sourceItems["main"].snapshot.UUID)
 	c.Assert(r.RefList("main").Len(), Equals, 0)
 
-	r, err = collection.ByPrefixDistribution("ppa", "precise")
+	r, err = collection.ByStoragePrefixDistribution("", "ppa", "precise")
 	c.Assert(err, IsNil)
 	c.Assert(r.sourceItems["main"].localRepo, IsNil)
 	c.Assert(s.collection.LoadComplete(r, s.factory), IsNil)
@@ -505,7 +556,7 @@ func (s *PublishedRepoCollectionSuite) TestLoadPre0_6(c *C) {
 	c.Assert(s.db.Put(s.repo1.RefKey(""), s.localRepo.RefList().Encode()), IsNil)
 
 	collection := NewPublishedRepoCollection(s.db)
-	repo, err := collection.ByPrefixDistribution("ppa", "anaconda")
+	repo, err := collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Check(err, IsNil)
 	c.Check(repo.Component, Equals, "")
 	c.Check(repo.SourceUUID, Equals, "")
@@ -548,20 +599,22 @@ func (s *PublishedRepoCollectionSuite) TestBySnapshot(c *C) {
 func (s *PublishedRepoCollectionSuite) TestByLocalRepo(c *C) {
 	c.Check(s.collection.Add(s.repo1), IsNil)
 	c.Check(s.collection.Add(s.repo4), IsNil)
+	c.Check(s.collection.Add(s.repo5), IsNil)
 
-	c.Check(s.collection.ByLocalRepo(s.localRepo), DeepEquals, []*PublishedRepo{s.repo4})
+	c.Check(s.collection.ByLocalRepo(s.localRepo), DeepEquals, []*PublishedRepo{s.repo4, s.repo5})
 }
 
 type PublishedRepoRemoveSuite struct {
 	PackageListMixinSuite
-	db                         database.Storage
-	factory                    *CollectionFactory
-	snapshotCollection         *SnapshotCollection
-	collection                 *PublishedRepoCollection
-	root                       string
-	publishedStorage           aptly.PublishedStorage
-	snap1                      *Snapshot
-	repo1, repo2, repo3, repo4 *PublishedRepo
+	db                                  database.Storage
+	factory                             *CollectionFactory
+	snapshotCollection                  *SnapshotCollection
+	collection                          *PublishedRepoCollection
+	root, root2                         string
+	provider                            *FakeStorageProvider
+	publishedStorage, publishedStorage2 *files.PublishedStorage
+	snap1                               *Snapshot
+	repo1, repo2, repo3, repo4, repo5   *PublishedRepo
 }
 
 var _ = Suite(&PublishedRepoRemoveSuite{})
@@ -576,16 +629,18 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 
 	s.snapshotCollection.Add(s.snap1)
 
-	s.repo1, _ = NewPublishedRepo("ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
-	s.repo2, _ = NewPublishedRepo("", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
-	s.repo3, _ = NewPublishedRepo("ppa", "meduza", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
-	s.repo4, _ = NewPublishedRepo("ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory)
+	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo3, _ = NewPublishedRepo("", "ppa", "meduza", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo4, _ = NewPublishedRepo("", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory)
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory)
 
 	s.collection = s.factory.PublishedRepoCollection()
 	s.collection.Add(s.repo1)
 	s.collection.Add(s.repo2)
 	s.collection.Add(s.repo3)
 	s.collection.Add(s.repo4)
+	s.collection.Add(s.repo5)
 
 	s.root = c.MkDir()
 	s.publishedStorage = files.NewPublishedStorage(s.root)
@@ -596,6 +651,15 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 	s.publishedStorage.MkDir("ppa/pool/contrib")
 	s.publishedStorage.MkDir("dists/anaconda")
 	s.publishedStorage.MkDir("pool/main")
+
+	s.root2 = c.MkDir()
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.publishedStorage2.MkDir("ppa/dists/osminog")
+	s.publishedStorage2.MkDir("ppa/pool/contrib")
+
+	s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{
+		"":            s.publishedStorage,
+		"files:other": s.publishedStorage2}}
 }
 
 func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
@@ -603,7 +667,7 @@ func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
-	s.repo1.RemoveFiles(s.publishedStorage, false, []string{}, nil)
+	s.repo1.RemoveFiles(s.provider, false, []string{}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
@@ -612,10 +676,12 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
-	s.repo1.RemoveFiles(s.publishedStorage, false, []string{"main"}, nil)
+	s.repo1.RemoveFiles(s.provider, false, []string{"main"}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
@@ -624,10 +690,12 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithTwoPools(c *C) {
-	s.repo1.RemoveFiles(s.publishedStorage, false, []string{"main", "contrib"}, nil)
+	s.repo1.RemoveFiles(s.provider, false, []string{"main", "contrib"}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
@@ -636,10 +704,12 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithTwoPools(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
-	s.repo1.RemoveFiles(s.publishedStorage, true, []string{"main"}, nil)
+	s.repo1.RemoveFiles(s.provider, true, []string{"main"}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
@@ -648,10 +718,12 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
-	s.repo2.RemoveFiles(s.publishedStorage, true, []string{"main"}, nil)
+	s.repo2.RemoveFiles(s.provider, true, []string{"main"}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
@@ -659,17 +731,19 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
-	err := s.collection.Remove(s.publishedStorage, "ppa", "anaconda", s.factory, nil)
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
 	c.Check(err, IsNil)
 
-	_, err = s.collection.ByPrefixDistribution("ppa", "anaconda")
+	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Check(err, ErrorMatches, ".*not found")
 
 	collection := NewPublishedRepoCollection(s.db)
-	_, err = collection.ByPrefixDistribution("ppa", "anaconda")
+	_, err = collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Check(err, ErrorMatches, ".*not found")
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
@@ -679,11 +753,13 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 
-	err = s.collection.Remove(s.publishedStorage, "ppa", "anaconda", s.factory, nil)
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
 	c.Check(err, ErrorMatches, ".*not found")
 
-	err = s.collection.Remove(s.publishedStorage, "ppa", "meduza", s.factory, nil)
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil)
 	c.Check(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
@@ -693,17 +769,19 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
-	err := s.collection.Remove(s.publishedStorage, ".", "anaconda", s.factory, nil)
+	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil)
 	c.Check(err, IsNil)
 
-	_, err = s.collection.ByPrefixDistribution(".", "anaconda")
+	_, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda")
 	c.Check(err, ErrorMatches, ".*not found")
 
 	collection := NewPublishedRepoCollection(s.db)
-	_, err = collection.ByPrefixDistribution(".", "anaconda")
+	_, err = collection.ByStoragePrefixDistribution("", ".", "anaconda")
 	c.Check(err, ErrorMatches, ".*not found")
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
@@ -713,4 +791,28 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) {
+	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	collection := NewPublishedRepoCollection(s.db)
+	_, err = collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
 }

deb/query.go

@@ -0,0 +1,267 @@
+package deb
+
+import (
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// PackageCatalog is abstraction on top of PackageCollection and PackageList
+type PackageCatalog interface {
+	Scan(q PackageQuery) (result *PackageList)
+	Search(dep Dependency, allMatches bool) (searchResults []*Package)
+	SearchSupported() bool
+	SearchByKey(arch, name, version string) (result *PackageList)
+}
+
+// PackageQuery is interface of predicate on Package
+type PackageQuery interface {
+	// Matches calculates match of condition against package
+	Matches(pkg *Package) bool
+	// Fast returns if search strategy is possible for this query
+	Fast(list PackageCatalog) bool
+	// Query performs search on package list
+	Query(list PackageCatalog) *PackageList
+	// String interface
+	String() string
+}
+
+// OrQuery is L | R
+type OrQuery struct {
+	L, R PackageQuery
+}
+
+// AndQuery is L , R
+type AndQuery struct {
+	L, R PackageQuery
+}
+
+// NotQuery is ! Q
+type NotQuery struct {
+	Q PackageQuery
+}
+
+// FieldQuery is generic request against field
+type FieldQuery struct {
+	Field    string
+	Relation int
+	Value    string
+	Regexp   *regexp.Regexp `codec:"-"`
+}
+
+// PkgQuery is search request against specific package
+type PkgQuery struct {
+	Pkg     string
+	Version string
+	Arch    string
+}
+
+// DependencyQuery is generic Debian-dependency like query
+type DependencyQuery struct {
+	Dep Dependency
+}
+
+// Matches if any of L, R matches
+func (q *OrQuery) Matches(pkg *Package) bool {
+	return q.L.Matches(pkg) || q.R.Matches(pkg)
+}
+
+// Fast is true only if both parts are fast
+func (q *OrQuery) Fast(list PackageCatalog) bool {
+	return q.L.Fast(list) && q.R.Fast(list)
+}
+
+// Query strategy depends on nodes
+func (q *OrQuery) Query(list PackageCatalog) (result *PackageList) {
+	if q.Fast(list) {
+		result = q.L.Query(list)
+		result.Append(q.R.Query(list))
+	} else {
+		result = list.Scan(q)
+	}
+	return
+}
+
+// String interface
+func (q *OrQuery) String() string {
+	return fmt.Sprintf("(%s) | (%s)", q.L, q.R)
+}
+
+// Matches if both of L, R matches
+func (q *AndQuery) Matches(pkg *Package) bool {
+	return q.L.Matches(pkg) && q.R.Matches(pkg)
+}
+
+// Fast is true if any of the parts are fast
+func (q *AndQuery) Fast(list PackageCatalog) bool {
+	return q.L.Fast(list) || q.R.Fast(list)
+}
+
+// Query strategy depends on nodes
+func (q *AndQuery) Query(list PackageCatalog) (result *PackageList) {
+	if !q.Fast(list) {
+		result = list.Scan(q)
+	} else {
+		if q.L.Fast(list) {
+			result = q.L.Query(list)
+			result = result.Scan(q.R)
+		} else {
+			result = q.R.Query(list)
+			result = result.Scan(q.L)
+		}
+	}
+	return
+}
+
+// String interface
+func (q *AndQuery) String() string {
+	return fmt.Sprintf("(%s), (%s)", q.L, q.R)
+}
+
+// Matches if not matches
+func (q *NotQuery) Matches(pkg *Package) bool {
+	return !q.Q.Matches(pkg)
+}
+
+// Fast is false
+func (q *NotQuery) Fast(list PackageCatalog) bool {
+	return false
+}
+
+// Query strategy is scan always
+func (q *NotQuery) Query(list PackageCatalog) (result *PackageList) {
+	result = list.Scan(q)
+	return
+}
+
+// String interface
+func (q *NotQuery) String() string {
+	return fmt.Sprintf("!(%s)", q.Q)
+}
+
+// Matches on generic field
+func (q *FieldQuery) Matches(pkg *Package) bool {
+	if q.Field == "$Version" {
+		return pkg.MatchesDependency(Dependency{Pkg: pkg.Name, Relation: q.Relation, Version: q.Value, Regexp: q.Regexp})
+	}
+	if q.Field == "$Architecture" && q.Relation == VersionEqual {
+		return pkg.MatchesArchitecture(q.Value)
+	}
+
+	field := pkg.GetField(q.Field)
+
+	switch q.Relation {
+	case VersionDontCare:
+		return field != ""
+	case VersionEqual:
+		return field == q.Value
+	case VersionGreater:
+		return field > q.Value
+	case VersionGreaterOrEqual:
+		return field >= q.Value
+	case VersionLess:
+		return field < q.Value
+	case VersionLessOrEqual:
+		return field <= q.Value
+	case VersionPatternMatch:
+		matched, err := filepath.Match(q.Value, field)
+		return err == nil && matched
+	case VersionRegexp:
+		if q.Regexp == nil {
+			q.Regexp = regexp.MustCompile(q.Value)
+		}
+		return q.Regexp.FindStringIndex(field) != nil
+
+	}
+	panic("unknown relation")
+}
+
+// Query runs iteration through list
+func (q *FieldQuery) Query(list PackageCatalog) (result *PackageList) {
+	result = list.Scan(q)
+	return
+}
+
+// Fast depends on the query
+func (q *FieldQuery) Fast(list PackageCatalog) bool {
+	return false
+}
+
+// String interface
+func (q *FieldQuery) String() string {
+	escape := func(val string) string {
+		if strings.IndexAny(val, "()|,!{} \t\n") != -1 {
+			return "'" + strings.Replace(strings.Replace(val, "\\", "\\\\", -1), "'", "\\'", -1) + "'"
+		}
+		return val
+	}
+
+	var op string
+	switch q.Relation {
+	case VersionEqual:
+		op = "="
+	case VersionGreater:
+		op = ">>"
+	case VersionLess:
+		op = "<<"
+	case VersionRegexp:
+		op = "~"
+	case VersionPatternMatch:
+		op = "%"
+	case VersionGreaterOrEqual:
+		op = ">="
+	case VersionLessOrEqual:
+		op = "<="
+	}
+	return fmt.Sprintf("%s (%s %s)", escape(q.Field), op, escape(q.Value))
+}
+
+// Matches on dependency condition
+func (q *DependencyQuery) Matches(pkg *Package) bool {
+	return pkg.MatchesDependency(q.Dep)
+}
+
+// Fast is always true for dependency query
+func (q *DependencyQuery) Fast(list PackageCatalog) bool {
+	return list.SearchSupported()
+}
+
+// Query runs PackageList.Search
+func (q *DependencyQuery) Query(list PackageCatalog) (result *PackageList) {
+	if q.Fast(list) {
+		result = NewPackageList()
+		for _, pkg := range list.Search(q.Dep, true) {
+			result.Add(pkg)
+		}
+	} else {
+		result = list.Scan(q)
+	}
+
+	return
+}
+
+// String interface
+func (q *DependencyQuery) String() string {
+	return q.Dep.String()
+}
+
+// Matches on specific properties
+func (q *PkgQuery) Matches(pkg *Package) bool {
+	return pkg.Name == q.Pkg && pkg.Version == q.Version && pkg.Architecture == q.Arch
+}
+
+// Fast is always true for package query
+func (q *PkgQuery) Fast(list PackageCatalog) bool {
+	return true
+}
+
+// Query looks up specific package
+func (q *PkgQuery) Query(list PackageCatalog) (result *PackageList) {
+	return list.SearchByKey(q.Arch, q.Pkg, q.Version)
+}
+
+// String interface
+func (q *PkgQuery) String() string {
+	return fmt.Sprintf("%s_%s_%s", q.Pkg, q.Version, q.Arch)
+}

deb/reflist.go

@@ -84,6 +84,14 @@ func (l *PackageRefList) ForEach(handler func([]byte) error) error {
 	return err
 }
 
+// Has checks whether package is part of reflist
+func (l *PackageRefList) Has(p *Package) bool {
+	key := p.Key("")
+
+	i := sort.Search(len(l.Refs), func(j int) bool { return bytes.Compare(l.Refs[j], key) >= 0 })
+	return i < len(l.Refs) && bytes.Compare(l.Refs[i], key) == 0
+}
+
 // Substract returns all packages in l that are not in r
 func (l *PackageRefList) Substract(r *PackageRefList) *PackageRefList {
 	result := &PackageRefList{Refs: make([][]byte, 0, 128)}

deb/reflist_test.go

@@ -128,6 +128,19 @@ func (s *PackageRefListSuite) TestPackageRefListForeach(c *C) {
 	c.Check(err, Equals, e)
 }
 
+func (s *PackageRefListSuite) TestHas(c *C) {
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	reflist := NewPackageRefListFromPackageList(s.list)
+
+	c.Check(reflist.Has(s.p1), Equals, true)
+	c.Check(reflist.Has(s.p3), Equals, true)
+	c.Check(reflist.Has(s.p5), Equals, true)
+	c.Check(reflist.Has(s.p2), Equals, true)
+	c.Check(reflist.Has(s.p6), Equals, false)
+}
+
 func (s *PackageRefListSuite) TestSubstract(c *C) {
 	r1 := []byte("r1")
 	r2 := []byte("r2")

deb/remote.go

@@ -16,9 +16,16 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 )
 
+// RemoteRepo statuses
+const (
+	MirrorIdle = iota
+	MirrorUpdating
+)
+
 // RemoteRepo represents remote (fetchable) Debian repository.
 //
 // Repostitory could be filtered when fetching by components, architectures
@@ -37,21 +44,35 @@ type RemoteRepo struct {
 	Architectures []string
 	// Should we download sources?
 	DownloadSources bool
+	// Should we download .udebs?
+	DownloadUdebs bool
 	// Meta-information about repository
 	Meta Stanza
 	// Last update date
 	LastDownloadDate time.Time
 	// Checksums for release files
 	ReleaseFiles map[string]utils.ChecksumInfo
+	// Filter for packages
+	Filter string
+	// FilterWithDeps to include dependencies from filter query
+	FilterWithDeps bool
+	// Status marks state of repository (being updated, no action)
+	Status int
+	// WorkerPID is PID of the process modifying the mirror (if any)
+	WorkerPID int
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
+	// Temporary list of package refs
+	tempPackageRefs *PackageRefList
 	// Parsed archived root
 	archiveRootURL *url.URL
+	// Current list of packages (filled while updating mirror)
+	packageList *PackageList
 }
 
 // NewRemoteRepo creates new instance of Debian remote repository with specified params
 func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string,
-	architectures []string, downloadSources bool) (*RemoteRepo, error) {
+	architectures []string, downloadSources bool, downloadUdebs bool) (*RemoteRepo, error) {
 	result := &RemoteRepo{
 		UUID:            uuid.New(),
 		Name:            name,
@@ -60,6 +81,7 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone
 		Components:      components,
 		Architectures:   architectures,
 		DownloadSources: downloadSources,
+		DownloadUdebs:   downloadUdebs,
 	}
 
 	err := result.prepare()
@@ -76,6 +98,9 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone
 		if len(result.Components) > 0 {
 			return nil, fmt.Errorf("components aren't supported for flat repos")
 		}
+		if result.DownloadUdebs {
+			return nil, fmt.Errorf("debian-installer udebs aren't supported for flat repos")
+		}
 		result.Components = nil
 	}
 
@@ -98,7 +123,10 @@ func (repo *RemoteRepo) prepare() error {
 func (repo *RemoteRepo) String() string {
 	srcFlag := ""
 	if repo.DownloadSources {
-		srcFlag = " [src]"
+		srcFlag += " [src]"
+	}
+	if repo.DownloadUdebs {
+		srcFlag += " [udeb]"
 	}
 	distribution := repo.Distribution
 	if distribution == "" {
@@ -127,6 +155,37 @@ func (repo *RemoteRepo) RefList() *PackageRefList {
 	return repo.packageRefs
 }
 
+// MarkAsUpdating puts current PID and sets status to updating
+func (repo *RemoteRepo) MarkAsUpdating() {
+	repo.Status = MirrorUpdating
+	repo.WorkerPID = os.Getpid()
+}
+
+// MarkAsIdle clears updating flag
+func (repo *RemoteRepo) MarkAsIdle() {
+	repo.Status = MirrorIdle
+	repo.WorkerPID = 0
+}
+
+// CheckLock returns error if mirror is being updated by another process
+func (repo *RemoteRepo) CheckLock() error {
+	if repo.Status == MirrorIdle || repo.WorkerPID == 0 {
+		return nil
+	}
+
+	p, err := os.FindProcess(repo.WorkerPID)
+	if err != nil {
+		return nil
+	}
+
+	err = p.Signal(syscall.Signal(0))
+	if err == nil {
+		return fmt.Errorf("mirror is locked by update operation, PID %d", repo.WorkerPID)
+	}
+
+	return nil
+}
+
 // ReleaseURL returns URL to Release* files in repo root
 func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
 	var path *url.URL
@@ -165,6 +224,13 @@ func (repo *RemoteRepo) SourcesURL(component string) *url.URL {
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
+// UdebURL returns URL of Packages files for given component and
+// architecture
+func (repo *RemoteRepo) UdebURL(component string, architecture string) *url.URL {
+	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/debian-installer/binary-%s/Packages", repo.Distribution, component, architecture)}
+	return repo.archiveRootURL.ResolveReference(path)
+}
+
 // PackageURL returns URL of package file relative to repository root
 // architecture
 func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
@@ -319,11 +385,13 @@ ok:
 	return nil
 }
 
-// Download downloads all repo files
-func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory, packagePool aptly.PackagePool, ignoreMismatch bool) error {
-	list := NewPackageList()
-
-	progress.Printf("Downloading & parsing package files...\n")
+// DownloadPackageIndexes downloads & parses package index files
+func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
+	ignoreMismatch bool) error {
+	if repo.packageList != nil {
+		panic("packageList != nil")
+	}
+	repo.packageList = NewPackageList()
 
 	// Download and parse all Packages & Source files
 	packagesURLs := [][]string{}
@@ -337,6 +405,9 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, co
 		for _, component := range repo.Components {
 			for _, architecture := range repo.Architectures {
 				packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
+				if repo.DownloadUdebs {
+					packagesURLs = append(packagesURLs, []string{repo.UdebURL(component, architecture).String(), "udeb"})
+				}
 			}
 			if repo.DownloadSources {
 				packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
@@ -373,13 +444,15 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, co
 
 			if kind == "binary" {
 				p = NewPackageFromControlFile(stanza)
+			} else if kind == "udeb" {
+				p = NewUdebPackageFromControlFile(stanza)
 			} else if kind == "source" {
 				p, err = NewSourcePackageFromControlFile(stanza)
 				if err != nil {
 					return err
 				}
 			}
-			err = list.Add(p)
+			err = repo.packageList.Add(p)
 			if err != nil {
 				return err
 			}
@@ -393,14 +466,30 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, co
 		progress.ShutdownBar()
 	}
 
-	progress.Printf("Building download queue...\n")
+	return nil
+}
 
-	// Build download queue
-	queued := make(map[string]PackageDownloadTask, list.Len())
-	count := 0
-	downloadSize := int64(0)
+// ApplyFilter applies filtering to already built PackageList
+func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQuery) (oldLen, newLen int, err error) {
+	repo.packageList.PrepareIndex()
 
-	err := list.ForEach(func(p *Package) error {
+	emptyList := NewPackageList()
+	emptyList.PrepareIndex()
+
+	oldLen = repo.packageList.Len()
+	repo.packageList, err = repo.packageList.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures)
+	if repo.packageList != nil {
+		newLen = repo.packageList.Len()
+	}
+	return
+}
+
+// BuildDownloadQueue builds queue, discards current PackageList
+func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool) (queue []PackageDownloadTask, downloadSize int64, err error) {
+	queue = make([]PackageDownloadTask, 0, repo.packageList.Len())
+	seen := make(map[string]struct{}, repo.packageList.Len())
+
+	err = repo.packageList.ForEach(func(p *Package) error {
 		list, err2 := p.DownloadList(packagePool)
 		if err2 != nil {
 			return err2
@@ -409,58 +498,31 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, co
 
 		for _, task := range list {
 			key := task.RepoURI + "-" + task.DestinationPath
-			_, found := queued[key]
+			_, found := seen[key]
 			if !found {
-				count++
+				queue = append(queue, task)
 				downloadSize += task.Checksums.Size
-				queued[key] = task
+				seen[key] = struct{}{}
 			}
 		}
 
 		return nil
 	})
 	if err != nil {
-		return fmt.Errorf("unable to build download queue: %s", err)
+		return
 	}
 
-	repo.packageRefs = NewPackageRefListFromPackageList(list)
+	repo.tempPackageRefs = NewPackageRefListFromPackageList(repo.packageList)
 	// free up package list, we don't need it after this point
-	list = nil
+	repo.packageList = nil
 
-	progress.Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
-
-	progress.InitBar(downloadSize, true)
-
-	// Download all package files
-	ch := make(chan error, len(queued))
-
-	for _, task := range queued {
-		d.DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
-	}
-
-	// We don't need queued after this point
-	queued = nil
-
-	// Wait for all downloads to finish
-	errors := make([]string, 0)
-
-	for count > 0 {
-		err = <-ch
-		if err != nil {
-			errors = append(errors, err.Error())
-		}
-		count--
-	}
-
-	progress.ShutdownBar()
-
-	if len(errors) > 0 {
-		return fmt.Errorf("download errors:\n  %s\n", strings.Join(errors, "\n  "))
-	}
+	return
+}
 
+// FinalizeDownload swaps for final value of package refs
+func (repo *RemoteRepo) FinalizeDownload() {
 	repo.LastDownloadDate = time.Now()
-
-	return nil
+	repo.packageRefs = repo.tempPackageRefs
 }
 
 // Encode does msgpack encoding of RemoteRepo
@@ -478,7 +540,43 @@ func (repo *RemoteRepo) Decode(input []byte) error {
 	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
 	err := decoder.Decode(repo)
 	if err != nil {
-		return err
+		if strings.HasPrefix(err.Error(), "codec.decoder: readContainerLen: Unrecognized descriptor byte: hex: 80") {
+			// probably it is broken DB from go < 1.2, try decoding w/o time.Time
+			var repo11 struct {
+				UUID             string
+				Name             string
+				ArchiveRoot      string
+				Distribution     string
+				Components       []string
+				Architectures    []string
+				DownloadSources  bool
+				Meta             Stanza
+				LastDownloadDate []byte
+				ReleaseFiles     map[string]utils.ChecksumInfo
+				Filter           string
+				FilterWithDeps   bool
+			}
+
+			decoder = codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
+			err2 := decoder.Decode(&repo11)
+			if err2 != nil {
+				return err
+			}
+
+			repo.UUID = repo11.UUID
+			repo.Name = repo11.Name
+			repo.ArchiveRoot = repo11.ArchiveRoot
+			repo.Distribution = repo11.Distribution
+			repo.Components = repo11.Components
+			repo.Architectures = repo11.Architectures
+			repo.DownloadSources = repo11.DownloadSources
+			repo.Meta = repo11.Meta
+			repo.ReleaseFiles = repo11.ReleaseFiles
+			repo.Filter = repo11.Filter
+			repo.FilterWithDeps = repo11.FilterWithDeps
+		} else {
+			return err
+		}
 	}
 	return repo.prepare()
 }

deb/remote_test.go

@@ -12,6 +12,7 @@ import (
 	"io/ioutil"
 	. "launchpad.net/gocheck"
 	"os"
+	"sort"
 )
 
 type NullVerifier struct {
@@ -79,8 +80,8 @@ type RemoteRepoSuite struct {
 var _ = Suite(&RemoteRepoSuite{})
 
 func (s *RemoteRepoSuite) SetUpTest(c *C) {
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false)
-	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false)
+	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false)
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress()
 	s.db, _ = database.OpenDB(c.MkDir())
@@ -96,7 +97,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
-	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false)
+	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(err, ErrorMatches, ".*hexadecimal escape in host.*")
 }
 
@@ -106,11 +107,11 @@ func (s *RemoteRepoSuite) TestFlatCreation(c *C) {
 	c.Check(s.flat.Architectures, IsNil)
 	c.Check(s.flat.Components, IsNil)
 
-	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false)
+	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false)
 	c.Check(flat2.IsFlat(), Equals, true)
 	c.Check(flat2.Distribution, Equals, "./binary/")
 
-	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false)
+	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false)
 	c.Check(err, ErrorMatches, "components aren't supported for flat repos")
 }
 
@@ -119,8 +120,9 @@ func (s *RemoteRepoSuite) TestString(c *C) {
 	c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./")
 
 	s.repo.DownloadSources = true
+	s.repo.DownloadUdebs = true
 	s.flat.DownloadSources = true
-	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src]")
+	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb]")
 	c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./ [src]")
 }
 
@@ -151,6 +153,10 @@ func (s *RemoteRepoSuite) TestBinaryURL(c *C) {
 	c.Assert(s.repo.BinaryURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/binary-amd64/Packages")
 }
 
+func (s *RemoteRepoSuite) TestUdebURL(c *C) {
+	c.Assert(s.repo.UdebURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/debian-installer/binary-amd64/Packages")
+}
+
 func (s *RemoteRepoSuite) TestSourcesURL(c *C) {
 	c.Assert(s.repo.SourcesURL("main").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources")
 }
@@ -209,13 +215,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*")
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "component xyz not available in repo.*")
 }
@@ -249,20 +255,22 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", errors.New("HTTP 404"))
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", errors.New("HTTP 404"))
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
-	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
 
-	err = s.repo.Download(s.progress, s.downloader, s.collectionFactory, s.packagePool, false)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.repo.FinalizeDownload()
 	c.Assert(s.repo.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	result, err := pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "amanda-client")
 }
 
@@ -279,32 +287,35 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", errors.New("HTTP 404"))
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", errors.New("HTTP 404"))
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
-	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
-	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc", "abc")
-	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz", "abcd")
-	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz", "abcde")
 
-	err = s.repo.Download(s.progress, s.downloader, s.collectionFactory, s.packagePool, false)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	q := make([]string, 4)
+	for i := range q {
+		q[i] = queue[i].RepoURI
+	}
+	sort.Strings(q)
+	c.Check(q[3], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+	c.Check(q[1], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc")
+	c.Check(q[2], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz")
+	c.Check(q[0], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz")
+
+	s.repo.FinalizeDownload()
 	c.Assert(s.repo.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	result, err := pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "amanda-client")
 
 	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
-
-	result, err = pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
 }
 
@@ -314,23 +325,25 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", errors.New("HTTP 404"))
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", errors.New("HTTP 404"))
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
-	downloader.ExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
 
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.Download(s.progress, downloader, s.collectionFactory, s.packagePool, false)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.flat.FinalizeDownload()
 	c.Assert(s.flat.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	result, err := pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "amanda-client")
 }
 
@@ -345,35 +358,39 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", errors.New("HTTP 404"))
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", errors.New("HTTP 404"))
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
-	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
-	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc", "abc")
-	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz", "abcd")
-	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz", "abcde")
 
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.Download(s.progress, downloader, s.collectionFactory, s.packagePool, false)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	q := make([]string, 4)
+	for i := range q {
+		q[i] = queue[i].RepoURI
+	}
+	sort.Strings(q)
+	c.Check(q[3], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+	c.Check(q[1], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc")
+	c.Check(q[2], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz")
+	c.Check(q[0], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz")
+
+	s.flat.FinalizeDownload()
 	c.Assert(s.flat.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	result, err := pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "amanda-client")
 
 	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
 
-	result, err = pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
 }
 
@@ -399,7 +416,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 	r, err := s.collection.ByName("yandex")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
@@ -417,7 +434,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 	r, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 
 	r, err = s.collection.ByUUID(repo.UUID)
@@ -426,7 +443,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection := NewRemoteRepoCollection(s.db)
@@ -447,7 +464,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	s.collection.Add(repo)
 
 	count := 0
@@ -469,10 +486,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestDrop(c *C) {
-	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	s.collection.Add(repo1)
 
-	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false)
+	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false)
 	s.collection.Add(repo2)
 
 	r1, _ := s.collection.ByUUID(repo1.UUID)

deb/snapshot.go

@@ -9,6 +9,7 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/ugorji/go/codec"
 	"log"
+	"strings"
 	"time"
 )
 
@@ -125,7 +126,36 @@ func (s *Snapshot) Encode() []byte {
 // Decode decodes msgpack representation into Snapshot
 func (s *Snapshot) Decode(input []byte) error {
 	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
-	return decoder.Decode(s)
+	err := decoder.Decode(s)
+	if err != nil {
+		if strings.HasPrefix(err.Error(), "codec.decoder: readContainerLen: Unrecognized descriptor byte: hex: 80") {
+			// probably it is broken DB from go < 1.2, try decoding w/o time.Time
+			var snapshot11 struct {
+				UUID string
+				Name string
+				CreatedAt []byte
+
+				SourceKind string
+				SourceIDs  []string
+				Description string
+			}
+
+			decoder = codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
+			err2 := decoder.Decode(&snapshot11)
+			if err2 != nil {
+				return err
+			}
+
+			s.UUID = snapshot11.UUID
+			s.Name = snapshot11.Name
+			s.SourceKind = snapshot11.SourceKind
+			s.SourceIDs = snapshot11.SourceIDs
+			s.Description = snapshot11.Description
+		} else {
+			return err
+		}
+	}
+	return nil
 }
 
 // SnapshotCollection does listing, updating/adding/deleting of Snapshots
@@ -178,7 +208,10 @@ func (collection *SnapshotCollection) Update(snapshot *Snapshot) error {
 	if err != nil {
 		return err
 	}
-	return collection.db.Put(snapshot.RefKey(), snapshot.packageRefs.Encode())
+	if snapshot.packageRefs != nil {
+		return collection.db.Put(snapshot.RefKey(), snapshot.packageRefs.Encode())
+	}
+	return nil
 }
 
 // LoadComplete loads additional information about snapshot

deb/snapshot_test.go

@@ -15,7 +15,7 @@ var _ = Suite(&SnapshotSuite{})
 
 func (s *SnapshotSuite) SetUpTest(c *C) {
 	s.SetUpPackages()
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	s.repo.packageRefs = s.reflist
 }
 
@@ -108,11 +108,11 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
 	s.collection = NewSnapshotCollection(s.db)
 	s.SetUpPackages()
 
-	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	s.repo1.packageRefs = s.reflist
 	s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1)
 
-	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false)
+	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false)
 	s.repo2.packageRefs = s.reflist
 	s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2)
 
@@ -192,7 +192,7 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) {
 	c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1})
 	c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2})
 
-	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false)
+	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false)
 
 	c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot{})
 }

deb/version.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 	"unicode"
@@ -178,6 +179,8 @@ const (
 	VersionEqual
 	VersionGreaterOrEqual
 	VersionGreater
+	VersionPatternMatch
+	VersionRegexp
 )
 
 // Dependency is a parsed version of Debian dependency to package
@@ -186,6 +189,7 @@ type Dependency struct {
 	Relation     int
 	Version      string
 	Architecture string
+	Regexp       *regexp.Regexp
 }
 
 // Hash calculates some predefined unique ID of Dependency
@@ -207,6 +211,10 @@ func (d *Dependency) String() string {
 		rel = ">="
 	case VersionLessOrEqual:
 		rel = "<="
+	case VersionPatternMatch:
+		rel = "%"
+	case VersionRegexp:
+		rel = "~"
 	case VersionDontCare:
 		return fmt.Sprintf("%s [%s]", d.Pkg, d.Architecture)
 	}

files/public.go

@@ -1,10 +1,12 @@
 package files
 
 import (
+	"fmt"
 	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/utils"
+	"io"
 	"os"
 	"path/filepath"
+	"syscall"
 )
 
 // PublishedStorage abstract file system with public dirs (published repos)
@@ -12,9 +14,10 @@ type PublishedStorage struct {
 	rootPath string
 }
 
-// Check interface
+// Check interfaces
 var (
-	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+	_ aptly.PublishedStorage      = (*PublishedStorage)(nil)
+	_ aptly.LocalPublishedStorage = (*PublishedStorage)(nil)
 )
 
 // NewPublishedStorage creates new instance of PublishedStorage which specified root
@@ -32,9 +35,26 @@ func (storage *PublishedStorage) MkDir(path string) error {
 	return os.MkdirAll(filepath.Join(storage.rootPath, path), 0755)
 }
 
-// CreateFile creates file for writing under public path
-func (storage *PublishedStorage) CreateFile(path string) (*os.File, error) {
-	return os.Create(filepath.Join(storage.rootPath, path))
+// PutFile puts file into published storage at specified path
+func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
+	var (
+		source, f *os.File
+		err       error
+	)
+	source, err = os.Open(sourceFilename)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	f, err = os.Create(filepath.Join(storage.rootPath, path))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = io.Copy(f, source)
+	return err
 }
 
 // Remove removes single file under public path
@@ -59,7 +79,8 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 // sourcePath is filepath to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool, sourcePath string) error {
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool,
+	sourcePath, sourceMD5 string, force bool) error {
 	// verify that package pool is local pool is filesystem pool
 	_ = sourcePool.(*PackagePool)
 
@@ -71,11 +92,38 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 		return err
 	}
 
-	_, err = os.Stat(filepath.Join(poolPath, baseName))
-	if err == nil { // already exists, skip
-		return nil
+	var dstStat, srcStat os.FileInfo
+
+	dstStat, err = os.Stat(filepath.Join(poolPath, baseName))
+	if err == nil {
+		// already exists, check source file
+		srcStat, err = os.Stat(sourcePath)
+		if err != nil {
+			// source file doesn't exist? problem!
+			return err
+		}
+
+		srcSys := srcStat.Sys().(*syscall.Stat_t)
+		dstSys := dstStat.Sys().(*syscall.Stat_t)
+
+		// source and destination inodes match, no need to link
+		if srcSys.Ino == dstSys.Ino {
+			return nil
+		}
+
+		// source and destination have different inodes, if !forced, this is fatal error
+		if !force {
+			return fmt.Errorf("error linking file to %s: file already exists and is different", filepath.Join(poolPath, baseName))
+		}
+
+		// forced, so remove destination
+		err = os.Remove(filepath.Join(poolPath, baseName))
+		if err != nil {
+			return err
+		}
 	}
 
+	// destination doesn't exist (or forced), create link
 	return os.Link(sourcePath, filepath.Join(poolPath, baseName))
 }
 
@@ -94,12 +142,12 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 		return nil
 	})
 
-	return result, err
-}
+	if err != nil && os.IsNotExist(err) {
+		// file path doesn't exist, consider it empty
+		return []string{}, nil
+	}
 
-// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
-func (storage *PublishedStorage) ChecksumsForFile(path string) (utils.ChecksumInfo, error) {
-	return utils.ChecksumsForFile(filepath.Join(storage.rootPath, path))
+	return result, err
 }
 
 // RenameFile renames (moves) file

files/public_test.go

@@ -32,13 +32,12 @@ func (s *PublishedStorageSuite) TestMkDir(c *C) {
 	c.Assert(err, IsNil)
 }
 
-func (s *PublishedStorageSuite) TestCreateFile(c *C) {
+func (s *PublishedStorageSuite) TesPutFile(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
-	file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file.Close()
 
 	_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
 	c.Assert(err, IsNil)
@@ -48,26 +47,27 @@ func (s *PublishedStorageSuite) TestFilelist(c *C) {
 	err := s.storage.MkDir("ppa/pool/main/a/ab/")
 	c.Assert(err, IsNil)
 
-	file, err := s.storage.CreateFile("ppa/pool/main/a/ab/a.deb")
+	err = s.storage.PutFile("ppa/pool/main/a/ab/a.deb", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file.Close()
 
-	file2, err := s.storage.CreateFile("ppa/pool/main/a/ab/b.deb")
+	err = s.storage.PutFile("ppa/pool/main/a/ab/b.deb", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file2.Close()
 
 	list, err := s.storage.Filelist("ppa/pool/main/")
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []string{"a/ab/a.deb", "a/ab/b.deb"})
+
+	list, err = s.storage.Filelist("ppa/pool/doenstexist/")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
 }
 
 func (s *PublishedStorageSuite) TestRenameFile(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
-	file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file.Close()
 
 	err = s.storage.RenameFile("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease")
 	c.Check(err, IsNil)
@@ -80,9 +80,8 @@ func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
-	file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file.Close()
 
 	err = s.storage.RemoveDirs("ppa/dists/", nil)
 
@@ -95,9 +94,8 @@ func (s *PublishedStorageSuite) TestRemove(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
-	file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file.Close()
 
 	err = s.storage.Remove("ppa/dists/squeeze/Release")
 
@@ -155,7 +153,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		err = ioutil.WriteFile(t.sourcePath, []byte("Contents"), 0644)
 		c.Assert(err, IsNil)
 
-		err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath)
+		err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, "", false)
 		c.Assert(err, IsNil)
 
 		st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename))
@@ -164,4 +162,31 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		info := st.Sys().(*syscall.Stat_t)
 		c.Check(int(info.Nlink), Equals, 2)
 	}
+
+	// test linking files to duplicate final name
+	sourcePath := filepath.Join(s.root, "pool/02/bc/mars-invaders_1.03.deb")
+	err := os.MkdirAll(filepath.Dir(sourcePath), 0755)
+	c.Assert(err, IsNil)
+
+	err = ioutil.WriteFile(sourcePath, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "", false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different")
+
+	st, err := os.Stat(sourcePath)
+	c.Assert(err, IsNil)
+
+	info := st.Sys().(*syscall.Stat_t)
+	c.Check(int(info.Nlink), Equals, 1)
+
+	// linking with force
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "", true)
+	c.Check(err, IsNil)
+
+	st, err = os.Stat(sourcePath)
+	c.Assert(err, IsNil)
+
+	info = st.Sys().(*syscall.Stat_t)
+	c.Check(int(info.Nlink), Equals, 2)
 }

http/download.go

@@ -1,11 +1,13 @@
 package http
 
 import (
+	"code.google.com/p/mxk/go1/flowcontrol"
 	"compress/bzip2"
 	"compress/gzip"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/go-ftp-protocol/protocol"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -21,14 +23,15 @@ var (
 
 // downloaderImpl is implementation of Downloader interface
 type downloaderImpl struct {
-	queue    chan *downloadTask
-	stop     chan bool
-	stopped  chan bool
-	pause    chan bool
-	unpause  chan bool
-	progress aptly.Progress
-	threads  int
-	client   *http.Client
+	queue     chan *downloadTask
+	stop      chan struct{}
+	stopped   chan struct{}
+	pause     chan struct{}
+	unpause   chan struct{}
+	progress  aptly.Progress
+	aggWriter io.Writer
+	threads   int
+	client    *http.Client
 }
 
 // downloadTask represents single item in queue
@@ -41,24 +44,31 @@ type downloadTask struct {
 }
 
 // NewDownloader creates new instance of Downloader which specified number
-// of threads
-func NewDownloader(threads int, progress aptly.Progress) aptly.Downloader {
+// of threads and download limit in bytes/sec
+func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.Downloader {
+	transport := *http.DefaultTransport.(*http.Transport)
+	transport.DisableCompression = true
+	transport.RegisterProtocol("ftp", &protocol.FTPRoundTripper{})
+
 	downloader := &downloaderImpl{
 		queue:    make(chan *downloadTask, 1000),
-		stop:     make(chan bool),
-		stopped:  make(chan bool),
-		pause:    make(chan bool),
-		unpause:  make(chan bool),
+		stop:     make(chan struct{}, threads),
+		stopped:  make(chan struct{}, threads),
+		pause:    make(chan struct{}),
+		unpause:  make(chan struct{}),
 		threads:  threads,
 		progress: progress,
 		client: &http.Client{
-			Transport: &http.Transport{
-				DisableCompression: true,
-				Proxy:              http.ProxyFromEnvironment,
-			},
+			Transport: &transport,
 		},
 	}
 
+	if downLimit > 0 {
+		downloader.aggWriter = flowcontrol.NewWriter(progress, downLimit)
+	} else {
+		downloader.aggWriter = progress
+	}
+
 	for i := 0; i < downloader.threads; i++ {
 		go downloader.process()
 	}
@@ -70,7 +80,7 @@ func NewDownloader(threads int, progress aptly.Progress) aptly.Downloader {
 // but doesn't process rest of queue
 func (downloader *downloaderImpl) Shutdown() {
 	for i := 0; i < downloader.threads; i++ {
-		downloader.stop <- true
+		downloader.stop <- struct{}{}
 	}
 
 	for i := 0; i < downloader.threads; i++ {
@@ -78,17 +88,24 @@ func (downloader *downloaderImpl) Shutdown() {
 	}
 }
 
+// Abort stops downloader but doesn't wait for downloader to stop
+func (downloader *downloaderImpl) Abort() {
+	for i := 0; i < downloader.threads; i++ {
+		downloader.stop <- struct{}{}
+	}
+}
+
 // Pause pauses task processing
 func (downloader *downloaderImpl) Pause() {
 	for i := 0; i < downloader.threads; i++ {
-		downloader.pause <- true
+		downloader.pause <- struct{}{}
 	}
 }
 
 // Resume resumes task processing
 func (downloader *downloaderImpl) Resume() {
 	for i := 0; i < downloader.threads; i++ {
-		downloader.unpause <- true
+		downloader.unpause <- struct{}{}
 	}
 }
 
@@ -114,10 +131,12 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 
 	resp, err := downloader.client.Get(task.url)
 	if err != nil {
-		task.result <- err
+		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
 	}
-	defer resp.Body.Close()
+	if resp.Body != nil {
+		defer resp.Body.Close()
+	}
 
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
 		task.result <- fmt.Errorf("HTTP code %d while fetching %s", resp.StatusCode, task.url)
@@ -126,7 +145,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 
 	err = os.MkdirAll(filepath.Dir(task.destination), 0755)
 	if err != nil {
-		task.result <- err
+		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
 	}
 
@@ -134,13 +153,13 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 
 	outfile, err := os.Create(temppath)
 	if err != nil {
-		task.result <- err
+		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
 	}
 	defer outfile.Close()
 
 	checksummer := utils.NewChecksumWriter()
-	writers := []io.Writer{outfile, downloader.progress}
+	writers := []io.Writer{outfile, downloader.aggWriter}
 
 	if task.expected.Size != -1 {
 		writers = append(writers, checksummer)
@@ -151,7 +170,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	_, err = io.Copy(w, resp.Body)
 	if err != nil {
 		os.Remove(temppath)
-		task.result <- err
+		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
 	}
 
@@ -182,7 +201,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	err = os.Rename(temppath, task.destination)
 	if err != nil {
 		os.Remove(temppath)
-		task.result <- err
+		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
 	}
 
@@ -194,7 +213,7 @@ func (downloader *downloaderImpl) process() {
 	for {
 		select {
 		case <-downloader.stop:
-			downloader.stopped <- true
+			downloader.stopped <- struct{}{}
 			return
 		case <-downloader.pause:
 			<-downloader.unpause

http/download_test.go

@@ -60,7 +60,7 @@ func (s *DownloaderSuite) TearDownTest(c *C) {
 func (s *DownloaderSuite) TestStartupShutdown(c *C) {
 	goroutines := runtime.NumGoroutine()
 
-	d := NewDownloader(10, s.progress)
+	d := NewDownloader(10, 100, s.progress)
 	d.Shutdown()
 
 	// wait for goroutines to shutdown
@@ -72,7 +72,7 @@ func (s *DownloaderSuite) TestStartupShutdown(c *C) {
 }
 
 func (s *DownloaderSuite) TestPauseResume(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 
 	d.Pause()
@@ -80,7 +80,7 @@ func (s *DownloaderSuite) TestPauseResume(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadOK(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -90,7 +90,7 @@ func (s *DownloaderSuite) TestDownloadOK(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -131,7 +131,7 @@ func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownload404(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -141,7 +141,7 @@ func (s *DownloaderSuite) TestDownload404(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadConnectError(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -151,7 +151,7 @@ func (s *DownloaderSuite) TestDownloadConnectError(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadFileError(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -161,7 +161,7 @@ func (s *DownloaderSuite) TestDownloadFileError(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadTemp(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 
 	f, err := DownloadTemp(d, s.url+"/test")
@@ -178,7 +178,7 @@ func (s *DownloaderSuite) TestDownloadTemp(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadTempWithChecksum(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 
 	f, err := DownloadTempWithChecksum(d, s.url+"/test", utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
@@ -191,7 +191,7 @@ func (s *DownloaderSuite) TestDownloadTempWithChecksum(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadTempError(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 
 	f, err := DownloadTemp(d, s.url+"/doesntexist")

http/fake.go

@@ -123,6 +123,10 @@ func (f *FakeDownloader) Download(url string, filename string, result chan<- err
 func (f *FakeDownloader) Shutdown() {
 }
 
+// Abort does nothing
+func (f *FakeDownloader) Abort() {
+}
+
 // Pause does nothing
 func (f *FakeDownloader) Pause() {
 }

http/http.go

@@ -1,2 +1,2 @@
-// Package http provides all HTTP-related operations
+// Package http provides all HTTP (and FTP)-related operations
 package http

main.go

@@ -1,38 +1,10 @@
 package main
 
 import (
-	"fmt"
 	"github.com/smira/aptly/cmd"
 	"os"
 )
 
 func main() {
-	defer func() {
-		if r := recover(); r != nil {
-			fatal, ok := r.(*cmd.FatalError)
-			if !ok {
-				panic(r)
-			}
-			fmt.Println("ERROR:", fatal.Message)
-			os.Exit(fatal.ReturnCode)
-		}
-	}()
-
-	command := cmd.RootCommand()
-
-	flags, args, err := command.ParseFlags(os.Args[1:])
-	if err != nil {
-		cmd.Fatal(err)
-	}
-
-	err = cmd.InitContext(flags)
-	if err != nil {
-		cmd.Fatal(err)
-	}
-	defer cmd.ShutdownContext()
-
-	err = command.Dispatch(args)
-	if err != nil {
-		cmd.Fatal(err)
-	}
+	os.Exit(cmd.Run(cmd.RootCommand(), os.Args[1:], true))
 }

man/aptly.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "APTLY" "1" "June 2014" "" ""
+.TH "APTLY" "1" "October 2014" "" ""
 .
 .SH "NAME"
 \fBaptly\fR \- Debian repository management tool
@@ -22,10 +22,10 @@ aptly has integrated help that matches contents of this manual page, to get help
 aptly is a tool to create partial and full mirrors of remote repositories, manage local repositories, filter them, merge, upgrade individual packages, take snapshots and publish them back as Debian repositories\.
 .
 .P
-aptly\(cqs goal is to establish repeatability and controlled changes in a package\-centric environment\. aptly allows to fix a set of packages in a repository, so that package installation and upgrade becomes deterministic\. At the same time aptly allows to perform controlled, fine\-grained changes in repository contents to transition your package environment to new version\.
+aptly\(cqs goal is to establish repeatability and controlled changes in a package\-centric environment\. aptly allows one to fix a set of packages in a repository, so that package installation and upgrade becomes deterministic\. At the same time aptly allows one to perform controlled, fine\-grained changes in repository contents to transition your package environment to new version\.
 .
 .SH "CONFIGURATION"
-aptly looks for configuration file in \fB/etc/aptly\.conf\fR and \fB~/\.aptly\.conf\fR, if no config file found, new one is created\. If \fB\-config=\fR flag is specified, aptly would use config file at specified location\. Also aptly needs root directory for database, package and published repository storage\. If not specified, directory defaults to \fB~/\.aptly\fR, it will be created if missing\.
+aptly looks for configuration file first in \fB~/\.aptly\.conf\fR then in \fB/etc/aptly\.conf\fR and, if no config file found, new one is created in home directory\. If \fB\-config=\fR flag is specified, aptly would use config file at specified location\. Also aptly needs root directory for database, package and published repository storage\. If not specified, directory defaults to \fB~/\.aptly\fR, it will be created if missing\.
 .
 .P
 Configuration file is stored in JSON format (default values shown below):
@@ -37,6 +37,7 @@ Configuration file is stored in JSON format (default values shown below):
 {
   "rootDir": "$HOME/\.aptly",
   "downloadConcurrency": 4,
+  "downloadSpeedLimit": 0,
   "architectures": [],
   "dependencyFollowSuggests": false,
   "dependencyFollowRecommends": false
@@ -46,7 +47,19 @@ Configuration file is stored in JSON format (default values shown below):
   "gpgDisableVerify": false,
   "downloadSourcePackages": false,
   "ppaDistributorID": "ubuntu",
-  "ppaCodename": ""
+  "ppaCodename": "",
+  "S3PublishEndpoints": {
+    "test": {
+      "region": "us\-east\-1",
+      "bucket": "repo",
+      "awsAccessKeyID": ""
+      "awsSecretAccessKey": "",
+      "prefix": "",
+      "acl": "public\-read",
+      "storageClass": "",
+      "encryptionMethod": "",
+      "plusWorkaround": false
+    }
 }
 .
 .fi
@@ -65,6 +78,10 @@ is root of directory storage to store database (\fBrootDir\fR/db), downloaded pa
 is a number of parallel download threads to use when downloading packages
 .
 .TP
+\fBdownloadSpeedLimit\fR
+limit in kbytes/sec on download speed while mirroring remote repositieis
+.
+.TP
 \fBarchitectures\fR
 is a list of architectures to process; if left empty defaults to all available architectures; could be overridden with option \fB\-architectures\fR
 .
@@ -100,8 +117,53 @@ if enabled, all mirrors created would have flag set to download source packages;
 \fBppaDistributorID\fR, \fBppaCodename\fR
 specifies paramaters for short PPA url expansion, if left blank they default to output of \fBlsb_release\fR command
 .
-.SH "PACKAGE SPEC"
-Some commands accept package specs to identify list of packages to process\. Package spec is a list of following search conditions:
+.TP
+\fBS3PublishEndpoints\fR
+configuration of Amazon S3 publishing endpoints (see below)
+.
+.SH "S3 PUBLISHING ENDPOINTS"
+aptly could be configured to publish repository directly to Amazon S3\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings:
+.
+.TP
+\fBregion\fR
+Amazon region for S3 bucket (e\.g\. \fBus\-east\-1\fR)
+.
+.TP
+\fBbucket\fR
+bucket name
+.
+.TP
+\fBprefix\fR
+(optional) do publishing under specified prefix in the bucket, defaults to no prefix (bucket root)
+.
+.TP
+\fBacl\fR
+(optional) assign ACL to published files (one of the canned ACLs in Amazon terminology)\. Useful values: \fBprivate\fR (default) or \fBpublic\-read\fR (public repository)\. Public repositories could be consumed by \fBapt\fR using HTTP endpoint (Amazon bucket should be configured for "website hosting"), for private repositories special apt S3 transport is required\.
+.
+.TP
+\fBawsAccessKeyID\fR, \fBawsSecretAccessKey\fR
+(optional) Amazon credentials to access S3 bucket\. If not supplied, environment variables \fBAWS_ACCESS_KEY_ID\fR and \fBAWS_SECRET_ACCESS_KEY\fR are used\.
+.
+.TP
+\fBstorageClass\fR
+(optional) Amazon S3 storage class, defaults to \fBSTANDARD\fR\. Other values available: \fBREDUCED_REDUNDANCY\fR (lower price, lower redundancy)
+.
+.TP
+\fBencryptionMethod\fR
+(optional) server\-side encryption method, defaults to none\. Currently the only available encryption method is \fBAES256\fR
+.
+.TP
+\fBplusWorkaround\fR
+(optional) workaround misbehavior in apt and Amazon S3 for files with \fB+\fR in filename by creating two copies of package files with \fB+\fR in filename: one original and another one with spaces instead of plus signs With \fBplusWorkaround\fR enabled, package files with plus sign would be stored twice\. aptly might not cleanup files with spaces when published repository is dropped or updated (switched) to new version of repository (snapshot)\.
+.
+.P
+In order to publish to S3, specify endpoint as \fBs3:endpoint\-name:\fR before publishing prefix on the command line, e\.g\.:
+.
+.P
+\fBaptly publish snapshot wheezy\-main s3:test:\fR
+.
+.SH "PACKAGE QUERY"
+Some commands accept package queries to identify list of packages to process\. Package query syntax almost matches \fBreprepro\fR query language\. Query consists of the following simple terms:
 .
 .TP
 direct package reference
@@ -109,14 +171,63 @@ reference to exaclty one package\. Format is identical to the way aptly lists pa
 .
 .TP
 dependency condition
-syntax follows Debian dependency specification: package_name followed by optional version specification and architecture limit\.
+syntax follows Debian dependency specification: package_name followed by optional version specification and architecture limit, e\.g: \fBmysql\-client (>= 3\.6)\fR\.
+.
+.TP
+query against package fields
+syntax is the same as for dependency conditions, but instead of package name field name is used, e\.g: \fBPriority (optional)\fR\.
+.
+.P
+Supported fields:
+.
+.IP "\[ci]" 4
+all field names from Debian package control files are supported except for \fBFilename\fR, \fBMD5sum\fR, \fBSHA1\fR, \fBSHA256\fR, \fBSize\fR, \fBFiles\fR, \fBChecksums\-SHA1\fR, \fBChecksums\-SHA256\fR\.
+.
+.IP "\[ci]" 4
+\fB$Source\fR is a name of source package (for binary packages)
+.
+.IP "\[ci]" 4
+\fB$SourceVersion\fR is a version of source package
+.
+.IP "\[ci]" 4
+\fB$Architecture\fR is \fBArchitecture\fR for binary packages and \fBsource\fR for source packages, when matching with equal (\fB=\fR) operator, package with \fBany\fR architecture matches all architectures but \fBsource\fR\.
+.
+.IP "\[ci]" 4
+\fB$Version\fR has the same value as \fBVersion\fR, but comparison operators use Debian version precedence rules
+.
+.IP "\[ci]" 4
+\fB$PackageType\fR is \fBdeb\fR for binary packages and \fBsource\fR for source packages
+.
+.IP "" 0
+.
+.P
+Operators:
+.
+.TP
+\fB=\fR
+strict match, default operator is no operator is given
+.
+.TP
+\fB>=\fR, \fB<=\fR, \fB=\fR, \fB>>\fR (strictly greater), \fB<<\fR (strictly less)
+lexicographical comparison for all fields and special rules when comparing package versions
+.
+.TP
+\fB%\fR
+pattern matching, like shell patterns, supported special symbols are: \fB[^]?*\fR, e\.g\.: \fB$Version (% 3\.5\-*)\fR
+.
+.TP
+\fB~\fR
+regular expression matching, e\.g\.: \fBName (~ \.*\-dev)\fR
+.
+.P
+Simple terms could be combined into more complex queries using operators \fB,\fR (and), \fB|\fR (or) and \fB!\fR (not), parentheses \fB()\fR are used to change operator precedence\. Match value could be enclosed in single (\fB\(cq\fR) or double (\fB"\fR) quotes if required to resolve ambiguity, quotes inside quoted string should escaped with slash (\fB\e\fR)\.
 .
 .P
 Examples:
 .
 .TP
 \fBmysql\-client\fR
-matches package mysql\-client of any version and architecture (including source)
+matches package mysql\-client of any version and architecture (including source), also matches packages that \fBProvide:\fR \fBmysql\-client\fR\.
 .
 .TP
 \fBmysql\-client (>= 3\.6)\fR
@@ -130,8 +241,20 @@ matches package \fBmysql\-client\fR on architecture \fBi386\fR, architecture \fB
 \fBmysql\-client (>= 3\.6) {i386}\fR
 version and architecture conditions combined\.
 .
+.TP
+\fBlibmysqlclient18_5\.5\.35\-rel33\.0\-611\.squeeze_amd64\fR
+direct package reference\.
+.
+.TP
+\fB$Source (nginx)\fR
+all binary packages with \fBnginx\fR as source package\.
+.
+.TP
+\fB!Name (~ \.*\-dev), mail\-transport, $Version (>= 3\.5)\fR
+matches all packages that provide \fBmail\-transport\fR with name that has no suffix \fB\-dev\fR and with version greater or equal to \fB3\.5\fR\.
+.
 .P
-When specified on command line, condition may have to be quoted according to shell rules, so that it stays single argument:
+When specified on command line, query may have to be quoted according to shell rules, so that it stays single argument:
 .
 .P
 \fBaptly repo import percona stable \(cqmysql\-client (>= 3\.6)\(cq\fR
@@ -166,7 +289,7 @@ when processing dependencies, follow Suggests
 \fBaptly\fR \fBmirror\fR \fBcreate\fR \fIname\fR \fIarchive url\fR \fIdistribution\fR [\fIcomponent1\fR \|\.\|\.\|\.]
 .
 .P
-Creates mirror \fIname\fR of remote repository, aptly supports both regular and flat Debian repositories exported via HTTP\. aptly would try download Release file from remote repository and verify its\(cq signature\. Command line format resembles apt utlitily sources\.list(5)\.
+Creates mirror \fIname\fR of remote repository, aptly supports both regular and flat Debian repositories exported via HTTP and FTP\. aptly would try download Release file from remote repository and verify its\(cq signature\. Command line format resembles apt utlitily sources\.list(5)\.
 .
 .P
 PPA urls could specified in short format:
@@ -184,6 +307,14 @@ $ aptly mirror create wheezy\-main http://mirror\.yandex\.ru/debian/ wheezy main
 Options:
 .
 .TP
+\-\fBfilter\fR=
+filter packages in mirror
+.
+.TP
+\-\fBfilter\-with\-deps\fR=false
+when filtering, include dependencies of matching packages as well
+.
+.TP
 \-\fBignore\-signatures\fR=false
 disable verification of Release file signatures
 .
@@ -195,6 +326,10 @@ gpg keyring to use when verifying Release file (could be specified multiple time
 \-\fBwith\-sources\fR=false
 download source packages in addition to binary packages
 .
+.TP
+\-\fBwith\-udebs\fR=false
+download \.udeb packages (Debian installer support)
+.
 .SH "LIST MIRRORS"
 \fBaptly\fR \fBmirror\fR \fBlist\fR
 .
@@ -268,6 +403,14 @@ $ aptly mirror update wheezy\-main
 Options:
 .
 .TP
+\-\fBdownload\-limit\fR=0
+limit download speed (kbytes/sec)
+.
+.TP
+\-\fBforce\fR=false
+force update mirror even if it is locked by another process
+.
+.TP
 \-\fBignore\-checksums\fR=false
 ignore checksum mismatches while downloading package files and metadata
 .
@@ -279,11 +422,80 @@ disable verification of Release file signatures
 \-\fBkeyring\fR=
 gpg keyring to use when verifying Release file (could be specified multiple times)
 .
+.SH "RENAMES MIRROR"
+\fBaptly\fR \fBmirror\fR \fBrename\fR \fIold\-name\fR \fInew\-name\fR
+.
+.P
+Command changes name of the mirror\.Mirror name should be unique\.
+.
+.P
+Example:
+.
+.P
+$ aptly mirror rename wheezy\-min wheezy\-main
+.
+.SH "EDIT MIRROR SETTINGS"
+\fBaptly\fR \fBmirror\fR \fBedit\fR \fIname\fR
+.
+.P
+Command edit allows one to change settings of mirror: filters, list of architectures\.
+.
+.P
+Example:
+.
+.P
+$ aptly mirror edit \-filter=nginx \-filter\-with\-deps some\-mirror
+.
+.P
+Options:
+.
+.TP
+\-\fBfilter\fR=
+filter packages in mirror
+.
+.TP
+\-\fBfilter\-with\-deps\fR=false
+when filtering, include dependencies of matching packages as well
+.
+.TP
+\-\fBwith\-sources\fR=false
+download source packages in addition to binary packages
+.
+.TP
+\-\fBwith\-udebs\fR=false
+download \.udeb packages (Debian installer support)
+.
+.SH "SEARCH MIRROR FOR PACKAGES MATCHING QUERY"
+\fBaptly\fR \fBmirror\fR \fBsearch\fR \fIname\fR \fIpackage\-query\fR
+.
+.P
+Command search displays list of packages in mirror that match package query
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly mirror search wheezy\-main \(cq$Architecture (i386), Name (% *\-dev)\(cq
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBwith\-deps\fR=false
+include dependencies into search results
+.
 .SH "ADD PACKAGES TO LOCAL REPOSITORY"
 \fBaptly\fR \fBrepo\fR \fBadd\fR \fIname\fR
 .
 .P
-Command adds packages to local repository from \.deb (binary packages) and \.dsc (source packages) files\. When importing from directory aptly would do recursive scan looking for all files matching \fI\.deb or\fR\.dsc patterns\. Every file discovered would be analyzed to extract metadata, package would then be created and added to the database\. Files would be imported to internal package pool\. For source packages, all required files are added automatically as well\. Extra files for source package should be in the same directory as *\.dsc file\.
+Command adds packages to local repository from \.deb, \.udeb (binary packages) and \.dsc (source packages) files\. When importing from directory aptly would do recursive scan looking for all files matching \fI\.[u]deb or\fR\.dsc patterns\. Every file discovered would be analyzed to extract metadata, package would then be created and added to the database\. Files would be imported to internal package pool\. For source packages, all required files are added automatically as well\. Extra files for source package should be in the same directory as *\.dsc file\.
 .
 .P
 Example:
@@ -295,14 +507,18 @@ $ aptly repo add testing myapp\-0\.1\.2\.deb incoming/
 Options:
 .
 .TP
+\-\fBforce\-replace\fR=false
+when adding package that conflicts with existing package, remove existing package
+.
+.TP
 \-\fBremove\-files\fR=false
 remove files that have been imported successfully into repository
 .
 .SH "COPY PACKAGES BETWEEN LOCAL REPOSITORIES"
-\fBaptly\fR \fBrepo\fR \fBcopy\fR \fIsrc\-name\fR \fIdst\-name\fR \fIpackage\-spec\fR \fB\|\.\|\.\|\.\fR
+\fBaptly\fR \fBrepo\fR \fBcopy\fR \fIsrc\-name\fR \fIdst\-name\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
 .
 .P
-Command copy copies packages matching \fIpackage\-spec\fR from local repo \fIsrc\-name\fR to local repo \fIdst\-name\fR\.
+Command copy copies packages matching \fIpackage\-query\fR from local repo \fIsrc\-name\fR to local repo \fIdst\-name\fR\.
 .
 .P
 Example:
@@ -371,7 +587,7 @@ force local repo deletion even if used by snapshots
 \fBaptly\fR \fBrepo\fR \fBedit\fR \fIname\fR
 .
 .P
-Command edit allows to change metadata of local repository: comment, default distribution and component\.
+Command edit allows one to change metadata of local repository: comment, default distribution and component\.
 .
 .P
 Example:
@@ -395,10 +611,10 @@ default component when publishing
 default distribution when publishing
 .
 .SH "IMPORT PACKAGES FROM MIRROR TO LOCAL REPOSITORY"
-\fBaptly\fR \fBrepo\fR \fBimport\fR \fIsrc\-mirror\fR \fIdst\-repo\fR \fIpackage\-spec\fR \fB\|\.\|\.\|\.\fR
+\fBaptly\fR \fBrepo\fR \fBimport\fR \fIsrc\-mirror\fR \fIdst\-repo\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
 .
 .P
-Command import looks up packages matching \fIpackage\-spec\fR in mirror \fIsrc\-mirror\fR and copies them to local repo \fIdst\-repo\fR\.
+Command import looks up packages matching \fIpackage\-query\fR in mirror \fIsrc\-mirror\fR and copies them to local repo \fIdst\-repo\fR\.
 .
 .P
 Example:
@@ -437,10 +653,10 @@ Options:
 display list in machine\-readable format
 .
 .SH "MOVE PACKAGES BETWEEN LOCAL REPOSITORIES"
-\fBaptly\fR \fBrepo\fR \fBmove\fR \fIsrc\-name\fR \fIdst\-name\fR \fIpackage\-spec\fR \fB\|\.\|\.\|\.\fR
+\fBaptly\fR \fBrepo\fR \fBmove\fR \fIsrc\-name\fR \fIdst\-name\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
 .
 .P
-Command move moves packages matching \fIpackage\-spec\fR from local repo \fIsrc\-name\fR to local repo \fIdst\-name\fR\.
+Command move moves packages matching \fIpackage\-query\fR from local repo \fIsrc\-name\fR to local repo \fIdst\-name\fR\.
 .
 .P
 Example:
@@ -460,10 +676,10 @@ don\(cqt move, just show what would be moved
 follow dependencies when processing package\-spec
 .
 .SH "REMOVE PACKAGES FROM LOCAL REPOSITORY"
-\fBaptly\fR \fBrepo\fR \fBremove\fR \fIname\fR \fIpackage\-spec\fR \fB\|\.\|\.\|\.\fR
+\fBaptly\fR \fBrepo\fR \fBremove\fR \fIname\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
 .
 .P
-Commands removes packages matching \fIpackage\-spec\fR from local repository \fIname\fR\. If removed packages are not referenced by other repos or snapshots, they can be removed completely (including files) by running \(cqaptly db cleanup\(cq\.
+Commands removes packages matching \fIpackage\-query\fR from local repository \fIname\fR\. If removed packages are not referenced by other repos or snapshots, they can be removed completely (including files) by running \(cqaptly db cleanup\(cq\.
 .
 .P
 Example:
@@ -494,6 +710,44 @@ Options:
 \-\fBwith\-packages\fR=false
 show list of packages
 .
+.SH "RENAMES LOCAL REPOSITORY"
+\fBaptly\fR \fBrepo\fR \fBrename\fR \fIold\-name\fR \fInew\-name\fR
+.
+.P
+Command changes name of the local repo\. Local repo name should be unique\.
+.
+.P
+Example:
+.
+.P
+$ aptly repo rename wheezy\-min wheezy\-main
+.
+.SH "SEARCH REPO FOR PACKAGES MATCHING QUERY"
+\fBaptly\fR \fBrepo\fR \fBsearch\fR \fIname\fR \fIpackage\-query\fR
+.
+.P
+Command search displays list of packages in local repository that match package query
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly repo search my\-software \(cq$Architecture (i386), Name (% *\-dev)\(cq
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBwith\-deps\fR=false
+include dependencies into search results
+.
 .SH "CREATES SNAPSHOT OF MIRROR (LOCAL REPOSITORY) CONTENTS"
 \fBaptly\fR \fBsnapshot\fR \fBcreate\fR \fIname\fR \fBfrom\fR \fBmirror\fR \fImirror\-name\fR \fB|\fR \fBfrom\fR \fBrepo\fR \fIrepo\-name\fR \fB|\fR \fBempty\fR
 .
@@ -531,6 +785,10 @@ Options:
 \-\fBraw\fR=false
 display list in machine\-readable format
 .
+.TP
+\-\fBsort\fR=name
+display list in \(cqname\(cq or creation \(cqtime\(cq order
+.
 .SH "SHOWS DETAILS ABOUT SNAPSHOT"
 \fBaptly\fR \fBsnapshot\fR \fBshow\fR \fIname\fR
 .
@@ -577,10 +835,10 @@ $ aptly snapshot verify wheezy\-main wheezy\-contrib wheezy\-non\-free
 .IP "" 0
 .
 .SH "PULL PACKAGES FROM ANOTHER SNAPSHOT"
-\fBaptly\fR \fBsnapshot\fR \fBpull\fR \fIname\fR \fIsource\fR \fIdestination\fR \fIpackage\-name\fR \fB\|\.\|\.\|\.\fR
+\fBaptly\fR \fBsnapshot\fR \fBpull\fR \fIname\fR \fIsource\fR \fIdestination\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
 .
 .P
-Command pull pulls new packages along with its\(cq dependencies to snapshot \fIname\fR from snapshot \fIsource\fR\. Pull can upgrade package version in \fIname\fR with versions from \fIsource\fR following dependencies\. New snapshot \fIdestination\fR is created as a result of this process\. Packages could be specified simply as \(cqpackage\-name\(cq or as dependency \(cqpackage\-name (>= version)\(cq\.
+Command pull pulls new packages along with its\(cq dependencies to snapshot \fIname\fR from snapshot \fIsource\fR\. Pull can upgrade package version in \fIname\fR with versions from \fIsource\fR following dependencies\. New snapshot \fIdestination\fR is created as a result of this process\. Packages could be specified simply as \(cqpackage\-name\(cq or as package queries\.
 .
 .P
 Example:
@@ -599,6 +857,10 @@ $ aptly snapshot pull wheezy\-main wheezy\-backports wheezy\-new\-xorg xorg\-ser
 Options:
 .
 .TP
+\-\fBall\-matches\fR=false
+pull all the packages that satisfy the dependency version requirements
+.
+.TP
 \-\fBdry\-run\fR=false
 don\(cqt create destination snapshot, just show what would be pulled
 .
@@ -692,11 +954,75 @@ Options:
 \-\fBforce\fR=false
 remove snapshot even if it was used as source for other snapshots
 .
-.SH "REMOVE PUBLISHED REPOSITORY"
-\fBaptly\fR \fBpublish\fR \fBdrop\fR \fIdistribution\fR [\fIprefix\fR]
+.SH "RENAMES SNAPSHOT"
+\fBaptly\fR \fBsnapshot\fR \fBrename\fR \fIold\-name\fR \fInew\-name\fR
 .
 .P
-Command removes whatever has been published under specified \fIprefix\fR and \fIdistribution\fR name\.
+Command changes name of the snapshot\. Snapshot name should be unique\.
+.
+.P
+Example:
+.
+.P
+$ aptly snapshot rename wheezy\-min wheezy\-main
+.
+.SH "SEARCH SNAPSHOT FOR PACKAGES MATCHING QUERY"
+\fBaptly\fR \fBsnapshot\fR \fBsearch\fR \fIname\fR \fIpackage\-query\fR
+.
+.P
+Command search displays list of packages in snapshot that match package query
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly snapshot search wheezy\-main \(cq$Architecture (i386), Name (% *\-dev)\(cq
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBwith\-deps\fR=false
+include dependencies into search results
+.
+.SH "FILTER PACKAGES IN SNAPSHOT PRODUCING ANOTHER SNAPSHOT"
+\fBaptly\fR \fBsnapshot\fR \fBfilter\fR \fIsource\fR \fIdestination\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
+.
+.P
+Command filter does filtering in snapshot \fIsource\fR, producing another snapshot \fIdestination\fR\. Packages could be specified simply as \(cqpackage\-name\(cq or as package queries\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly snapshot filter wheezy\-main wheezy\-required \(cqPriorioty (required)\(cq
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBwith\-deps\fR=false
+include dependent packages as well
+.
+.SH "REMOVE PUBLISHED REPOSITORY"
+\fBaptly\fR \fBpublish\fR \fBdrop\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR]
+.
+.P
+Command removes whatever has been published under specified \fIprefix\fR, publishing \fIendpoint\fR and \fIdistribution\fR name\.
 .
 .P
 Example:
@@ -738,7 +1064,7 @@ Options:
 display list in machine\-readable format
 .
 .SH "PUBLISH LOCAL REPOSITORY"
-\fBaptly\fR \fBpublish\fR \fBrepo\fR \fIname\fR [\fIprefix\fR]
+\fBaptly\fR \fBpublish\fR \fBrepo\fR \fIname\fR [[\fIendpoint\fR:]\fIprefix\fR]
 .
 .P
 Command publishes current state of local repository ready to be consumed by apt tools\. Published repostiories appear under rootDir/public directory\. Valid GPG key is required for publishing\.
@@ -784,6 +1110,10 @@ component name to publish (for multi\-component publishing, separate components
 distribution name to publish
 .
 .TP
+\-\fBforce\-overwrite\fR=false
+overwrite files in package pool in case of mismatch
+.
+.TP
 \-\fBgpg\-key\fR=
 GPG key ID to use when signing the release
 .
@@ -800,6 +1130,14 @@ label to publish
 origin name to publish
 .
 .TP
+\-\fBpassphrase\fR=
+GPG passhprase for the key (warning: could be insecure)
+.
+.TP
+\-\fBpassphrase\-file\fR=
+GPG passhprase\-file for the key (warning: could be insecure)
+.
+.TP
 \-\fBsecret\-keyring\fR=
 GPG secret keyring to use (instead of default)
 .
@@ -808,7 +1146,7 @@ GPG secret keyring to use (instead of default)
 don\(cqt sign Release files with GPG
 .
 .SH "PUBLISH SNAPSHOT"
-\fBaptly\fR \fBpublish\fR \fBsnapshot\fR \fIname\fR [\fIprefix\fR]
+\fBaptly\fR \fBpublish\fR \fBsnapshot\fR \fIname\fR [[\fIendpoint\fR:]\fIprefix\fR]
 .
 .P
 Command publishes snapshot as Debian repository ready to be consumed by apt tools\. Published repostiories appear under rootDir/public directory\. Valid GPG key is required for publishing\.
@@ -851,6 +1189,10 @@ component name to publish (for multi\-component publishing, separate components
 distribution name to publish
 .
 .TP
+\-\fBforce\-overwrite\fR=false
+overwrite files in package pool in case of mismatch
+.
+.TP
 \-\fBgpg\-key\fR=
 GPG key ID to use when signing the release
 .
@@ -867,6 +1209,14 @@ label to publish
 origin name to publish
 .
 .TP
+\-\fBpassphrase\fR=
+GPG passhprase for the key (warning: could be insecure)
+.
+.TP
+\-\fBpassphrase\-file\fR=
+GPG passhprase\-file for the key (warning: could be insecure)
+.
+.TP
 \-\fBsecret\-keyring\fR=
 GPG secret keyring to use (instead of default)
 .
@@ -875,7 +1225,7 @@ GPG secret keyring to use (instead of default)
 don\(cqt sign Release files with GPG
 .
 .SH "UPDATE PUBLISHED REPOSITORY BY SWITCHING TO NEW SNAPSHOT"
-\fBaptly\fR \fBpublish\fR \fBswitch\fR \fIdistribution\fR [\fIprefix\fR] \fInew\-snapshot\fR
+\fBaptly\fR \fBpublish\fR \fBswitch\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR] \fInew\-snapshot\fR
 .
 .P
 Command switches in\-place published repository with new snapshot contents\. All publishing parameters are preserved (architecture list, distribution, component)\.
@@ -914,6 +1264,10 @@ Options:
 component names to update (for multi\-component publishing, separate components with commas)
 .
 .TP
+\-\fBforce\-overwrite\fR=false
+overwrite files in package pool in case of mismatch
+.
+.TP
 \-\fBgpg\-key\fR=
 GPG key ID to use when signing the release
 .
@@ -922,6 +1276,14 @@ GPG key ID to use when signing the release
 GPG keyring to use (instead of default)
 .
 .TP
+\-\fBpassphrase\fR=
+GPG passhprase for the key (warning: could be insecure)
+.
+.TP
+\-\fBpassphrase\-file\fR=
+GPG passhprase\-file for the key (warning: could be insecure)
+.
+.TP
 \-\fBsecret\-keyring\fR=
 GPG secret keyring to use (instead of default)
 .
@@ -930,7 +1292,7 @@ GPG secret keyring to use (instead of default)
 don\(cqt sign Release files with GPG
 .
 .SH "UPDATE PUBLISHED LOCAL REPOSITORY"
-\fBaptly\fR \fBpublish\fR \fBupdate\fR \fIdistribution\fR [\fIprefix\fR]
+\fBaptly\fR \fBpublish\fR \fBupdate\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR]
 .
 .P
 Command re\-publishes (updates) published local repository\. \fIdistribution\fR and \fIprefix\fR should be occupied with local repository published using command aptly publish repo\. Update happens in\-place with minimum possible downtime for published repository\.
@@ -955,6 +1317,10 @@ $ aptly publish update wheezy ppa
 Options:
 .
 .TP
+\-\fBforce\-overwrite\fR=false
+overwrite files in package pool in case of mismatch
+.
+.TP
 \-\fBgpg\-key\fR=
 GPG key ID to use when signing the release
 .
@@ -963,6 +1329,14 @@ GPG key ID to use when signing the release
 GPG keyring to use (instead of default)
 .
 .TP
+\-\fBpassphrase\fR=
+GPG passhprase for the key (warning: could be insecure)
+.
+.TP
+\-\fBpassphrase\-file\fR=
+GPG passhprase\-file for the key (warning: could be insecure)
+.
+.TP
 \-\fBsecret\-keyring\fR=
 GPG secret keyring to use (instead of default)
 .
@@ -970,6 +1344,55 @@ GPG secret keyring to use (instead of default)
 \-\fBskip\-signing\fR=false
 don\(cqt sign Release files with GPG
 .
+.SH "SEARCH FOR PACKAGES MATCHING QUERY"
+\fBaptly\fR \fBpackage\fR \fBsearch\fR \fIpackage\-query\fR
+.
+.P
+Command search displays list of packages in whole DB that match package query
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly package search \(cq$Architecture (i386), Name (% *\-dev)\(cq
+.
+.fi
+.
+.IP "" 0
+.
+.SH "SHOW DETAILS ABOUT PACKAGES MATCING QUERY"
+\fBaptly\fR \fBpackage\fR \fBshow\fR \fIpackage\-query\fR
+.
+.P
+Command shows displays detailed meta\-information about packages matching query\. Information from Debian control file is displayed\. Optionally information about package files and inclusion into mirrors/snapshots/local repos is shown\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly package show nginx\-light_1\.2\.1\-2\.2+wheezy2_i386\(cq
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBwith\-files\fR=false
+display information about files from package pool
+.
+.TP
+\-\fBwith\-references\fR=false
+display information about mirrors, snapshots and local repos referencing this package
+.
 .SH "CLEANUP DB AND PACKAGE POOL"
 \fBaptly\fR \fBdb\fR \fBcleanup\fR
 .

man/aptly.1.ronn.tmpl

@@ -18,8 +18,9 @@ aptly has integrated help that matches contents of this manual page, to get help
 
 ## CONFIGURATION
 
-aptly looks for configuration file in `/etc/aptly.conf` and `~/.aptly.conf`, if no config file
-found, new one is created. If `-config=` flag is specified, aptly would use config file at specified
+aptly looks for configuration file first in `~/.aptly.conf` then
+in `/etc/aptly.conf` and, if no config file found, new one is created in
+home directory. If `-config=` flag is specified, aptly would use config file at specified
 location. Also aptly needs root directory for database, package and published repository storage.
 If not specified, directory defaults to `~/.aptly`, it will be created if missing.
 
@@ -28,6 +29,7 @@ Configuration file is stored in JSON format (default values shown below):
     {
       "rootDir": "$HOME/.aptly",
       "downloadConcurrency": 4,
+      "downloadSpeedLimit": 0,
       "architectures": [],
       "dependencyFollowSuggests": false,
       "dependencyFollowRecommends": false
@@ -37,7 +39,19 @@ Configuration file is stored in JSON format (default values shown below):
       "gpgDisableVerify": false,
       "downloadSourcePackages": false,
       "ppaDistributorID": "ubuntu",
-      "ppaCodename": ""
+      "ppaCodename": "",
+      "S3PublishEndpoints": {
+        "test": {
+          "region": "us-east-1",
+          "bucket": "repo",
+          "awsAccessKeyID": ""
+          "awsSecretAccessKey": "",
+          "prefix": "",
+          "acl": "public-read",
+          "storageClass": "",
+          "encryptionMethod": "",
+          "plusWorkaround": false
+        }
     }
 
 Options:
@@ -49,6 +63,9 @@ Options:
   * `downloadConcurrency`:
     is a number of parallel download threads to use when downloading packages
 
+  * `downloadSpeedLimit`:
+    limit in kbytes/sec on download speed while mirroring remote repositieis
+
   * `architectures`:
     is a list of architectures to process; if left empty defaults to all available architectures; could be
     overridden with option `-architectures`
@@ -81,10 +98,57 @@ Options:
     specifies paramaters for short PPA url expansion, if left blank they default
     to output of `lsb_release` command
 
-## PACKAGE SPEC
+  * `S3PublishEndpoints`:
+    configuration of Amazon S3 publishing endpoints (see below)
 
-Some commands accept package specs to identify list of packages to process.
-Package spec is a list of following search conditions:
+## S3 PUBLISHING ENDPOINTS
+
+aptly could be configured to publish repository directly to Amazon S3. First, publishing
+endpoints should be described in aptly configuration file. Each endpoint has name
+and associated settings:
+
+   * `region`:
+     Amazon region for S3 bucket (e.g. `us-east-1`)
+   * `bucket`:
+     bucket name
+   * `prefix`:
+     (optional) do publishing under specified prefix in the bucket, defaults to
+     no prefix (bucket root)
+   * `acl`:
+     (optional) assign ACL to published files (one of the canned ACLs in Amazon
+     terminology). Useful values: `private` (default) or `public-read` (public
+     repository). Public repositories could be consumed by `apt` using
+     HTTP endpoint (Amazon bucket should be configured for "website hosting"),
+     for private repositories special apt S3 transport is required.
+   * `awsAccessKeyID`, `awsSecretAccessKey`:
+     (optional) Amazon credentials to access S3 bucket. If not supplied,
+     environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+     are used.
+   * `storageClass`:
+     (optional) Amazon S3 storage class, defaults to `STANDARD`. Other values
+     available: `REDUCED_REDUNDANCY` (lower price, lower redundancy)
+   * `encryptionMethod`:
+     (optional) server-side encryption method, defaults to none. Currently
+     the only available encryption method is `AES256`
+   * `plusWorkaround`:
+     (optional) workaround misbehavior in apt and Amazon S3
+     for files with `+` in filename by
+     creating two copies of package files with `+` in filename: one original
+     and another one with spaces instead of plus signs
+     With `plusWorkaround` enabled, package files with plus sign
+     would be stored twice. aptly might not cleanup files with spaces when published
+     repository is dropped or updated (switched) to new version of repository (snapshot).
+
+In order to publish to S3, specify endpoint as `s3:endpoint-name:` before
+publishing prefix on the command line, e.g.:
+
+  `aptly publish snapshot wheezy-main s3:test:`
+
+## PACKAGE QUERY
+
+Some commands accept package queries to identify list of packages to process.
+Package query syntax almost matches `reprepro` query language. Query consists of
+the following simple terms:
 
   * direct package reference:
     reference to exaclty one package. Format is identical to the way aptly lists packages in
@@ -92,12 +156,49 @@ Package spec is a list of following search conditions:
     e.g.: `libmysqlclient18_5.5.35-rel33.0-611.squeeze_amd64`
 
   * dependency condition:
-    syntax follows Debian dependency specification: package_name followed by optional version specification and architecture limit.
+    syntax follows Debian dependency specification: package_name followed by optional version specification
+    and architecture limit, e.g: `mysql-client (>= 3.6)`.
+
+  * query against package fields:
+    syntax is the same as for dependency conditions, but instead of package name field name is used, e.g:
+    `Priority (optional)`.
+
+Supported fields:
+
+  * all field names from Debian package control files are supported except for `Filename`, `MD5sum`,
+    `SHA1`, `SHA256`, `Size`, `Files`, `Checksums-SHA1`, `Checksums-SHA256`.
+  * `$Source` is a name of source package (for binary packages)
+  * `$SourceVersion` is a version of source package
+  * `$Architecture` is `Architecture` for binary packages and `source` for source packages,
+     when matching with equal (`=`) operator, package with `any` architecture matches all architectures
+     but `source`.
+  * `$Version` has the same value as `Version`, but comparison operators use Debian
+     version precedence rules
+  * `$PackageType` is `deb` for binary packages and `source` for source packages
+
+Operators:
+
+  * `=`:
+    strict match, default operator is no operator is given
+  * `>=`, `<=`, `=`, `>>` (strictly greater), `<<` (strictly less):
+    lexicographical comparison for all fields and special rules when comparing package versions
+  * `%`:
+    pattern matching, like shell patterns, supported special symbols are: `[^]?*`, e.g.:
+    `$Version (% 3.5-*)`
+  * `~`:
+    regular expression matching, e.g.:
+    `Name (~ .*-dev)`
+
+Simple terms could be combined into more complex queries using operators `,` (and), `|` (or) and
+`!` (not), parentheses `()` are used to change operator precedence. Match value could be
+enclosed in single (`'`) or double (`"`) quotes if required to resolve ambiguity, quotes
+inside quoted string should escaped with slash (`\`).
 
 Examples:
 
   * `mysql-client`:
-     matches package mysql-client of any version and architecture (including source)
+     matches package mysql-client of any version and architecture (including source), also
+     matches packages that `Provide:` `mysql-client`.
 
   * `mysql-client (>= 3.6)`:
      matches package mysql-client with version greater or equal to 3.6. Valid operators for
@@ -107,9 +208,19 @@ Examples:
      matches package `mysql-client` on architecture `i386`, architecture `all` matches all architectures but source.
 
   * `mysql-client (>= 3.6) {i386}`:
-      version and architecture conditions combined.
+    version and architecture conditions combined.
 
-When specified on command line, condition may have to be quoted according to shell rules, so that it stays single argument:
+  * `libmysqlclient18_5.5.35-rel33.0-611.squeeze_amd64`:
+    direct package reference.
+
+  * `$Source (nginx)`:
+    all binary packages with `nginx` as source package.
+
+  * `!Name (~ .*-dev), mail-transport, $Version (>= 3.5)`:
+    matches all packages that provide `mail-transport` with name that has no suffix `-dev` and
+    with version greater or equal to `3.5`.
+
+When specified on command line, query may have to be quoted according to shell rules, so that it stays single argument:
 
   `aptly repo import percona stable 'mysql-client (>= 3.6)'`
 
@@ -126,6 +237,8 @@ When specified on command line, condition may have to be quoted according to she
 
 {{template "command" findCommand . "publish"}}
 
+{{template "command" findCommand . "package"}}
+
 {{template "command" findCommand . "db"}}
 
 {{template "command" findCommand . "serve"}}

query/lex.go

@@ -0,0 +1,247 @@
+package query
+
+import (
+	"fmt"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// itemType identifies the type of lex items.
+type itemType int
+
+const eof = -1
+
+const (
+	itemNull  itemType = iota
+	itemError          // error occurred;
+	// value is text of error
+	itemEOF
+	itemLeftParen  // (
+	itemRightParen // )
+	itemOr         // |
+	itemAnd        // ,
+	itemNot        // !
+	itemLt         // <<
+	itemLtEq       // <=, <
+	itemGt         // >>
+	itemGtEq       // >=, >
+	itemEq         // =
+	itemPatMatch   // %
+	itemRegexp     // ~
+	itemLeftCurly  // {
+	itemRightCurly // }
+	itemString
+)
+
+// item represents a token returned from the scanner.
+type item struct {
+	typ itemType // Type, such as itemNumber.
+	val string   // Value, such as "23.2".
+}
+
+func (i item) String() string {
+	if i.typ == itemString {
+		return fmt.Sprintf("%#v", i.val)
+	}
+	if i.typ == itemEOF {
+		return "<EOL>"
+	}
+	if i.typ == itemError {
+		return fmt.Sprintf("error: %s", i.val)
+	}
+	if i.typ == itemNull {
+		return "<NULL>"
+	}
+	return i.val
+}
+
+// stateFn represents the state of the scanner
+// as a function that returns the next state.
+type stateFn func(*lexer) stateFn
+
+// lexer holds the state of the scanner.
+type lexer struct {
+	name  string    // used only for error reports.
+	input string    // the string being scanned.
+	start int       // start position of this item.
+	pos   int       // current position in the input.
+	width int       // width of last rune read from input.
+	items chan item // channel of scanned items.
+	last  item
+}
+
+func lex(name, input string) (*lexer, chan item) {
+	l := &lexer{
+		name:  name,
+		input: input,
+		items: make(chan item),
+	}
+	go l.run() // Concurrently run state machine.
+	return l, l.items
+}
+
+// emit passes an item back to the client.
+func (l *lexer) emit(t itemType) {
+	l.items <- item{t, l.input[l.start:l.pos]}
+	l.start = l.pos
+}
+
+// run lexes the input by executing state functions until
+// the state is nil.
+func (l *lexer) run() {
+	for state := lexMain; state != nil; {
+		state = state(l)
+	}
+	close(l.items) // No more tokens will be delivered.
+}
+
+// next returns the next rune in the input.
+func (l *lexer) next() (r rune) {
+	if l.pos >= len(l.input) {
+		l.width = 0
+		return eof
+	}
+	r, l.width =
+		utf8.DecodeRuneInString(l.input[l.pos:])
+	l.pos += l.width
+	return r
+}
+
+// ignore skips over the pending input before this point.
+func (l *lexer) ignore() {
+	l.start = l.pos
+}
+
+// backup steps back one rune.
+// Can be called only once per call of next.
+func (l *lexer) backup() {
+	l.pos -= l.width
+}
+
+// peek returns but does not consume
+// the next rune in the input.
+func (l *lexer) peek() rune {
+	r := l.next()
+	l.backup()
+	return r
+}
+
+func (l *lexer) Current() item {
+	if l.last.typ == 0 {
+		l.last = <-l.items
+	}
+
+	return l.last
+}
+
+func (l *lexer) Consume() {
+	l.last = <-l.items
+}
+
+// error returns an error token and terminates the scan
+// by passing back a nil pointer that will be the next
+// state, terminating l.run.
+func (l *lexer) errorf(format string, args ...interface{}) stateFn {
+	l.items <- item{
+		itemError,
+		fmt.Sprintf(format, args...),
+	}
+	return nil
+}
+
+func lexMain(l *lexer) stateFn {
+	switch r := l.next(); {
+	case r == eof:
+		l.emit(itemEOF)
+		return nil
+	case unicode.IsSpace(r):
+		l.ignore()
+	case r == '(':
+		l.emit(itemLeftParen)
+	case r == ')':
+		l.emit(itemRightParen)
+	case r == '{':
+		l.emit(itemLeftCurly)
+	case r == '}':
+		l.emit(itemRightCurly)
+	case r == '|':
+		l.emit(itemOr)
+	case r == ',':
+		l.emit(itemAnd)
+	case r == '!':
+		l.emit(itemNot)
+	case r == '<':
+		r2 := l.next()
+		if r2 == '<' {
+			l.emit(itemLt)
+		} else if r2 == '=' {
+			l.emit(itemLtEq)
+		} else {
+			l.backup()
+			l.emit(itemLtEq)
+		}
+	case r == '>':
+		r2 := l.next()
+		if r2 == '>' {
+			l.emit(itemGt)
+		} else if r2 == '=' {
+			l.emit(itemGtEq)
+		} else {
+			l.backup()
+			l.emit(itemGtEq)
+		}
+	case r == '=':
+		l.emit(itemEq)
+	case r == '%':
+		l.emit(itemPatMatch)
+	case r == '~':
+		l.emit(itemRegexp)
+	default:
+		l.backup()
+		return lexString
+	}
+
+	return lexMain
+}
+
+func lexString(l *lexer) stateFn {
+	r := l.next()
+	// quoted string
+	if r == '"' || r == '\'' {
+		quote := r
+		result := ""
+		l.ignore()
+		for {
+			r = l.next()
+			if r == quote {
+				l.ignore()
+				l.items <- item{itemString, result}
+				return lexMain
+			}
+			if r == '\\' {
+				r = l.next()
+			}
+			if r == eof {
+				return l.errorf("unexpected eof in quoted string")
+			}
+			result = result + string(r)
+		}
+	} else {
+		// unquoted string
+		for {
+			if unicode.IsSpace(r) || strings.IndexRune("()|,!{}", r) > 0 {
+				l.backup()
+				l.emit(itemString)
+				return lexMain
+			}
+
+			if r == eof {
+				l.emit(itemString)
+				l.emit(itemEOF)
+				return nil
+			}
+			r = l.next()
+		}
+	}
+}

query/lex_test.go

@@ -0,0 +1,57 @@
+package query
+
+import (
+	"fmt"
+	. "launchpad.net/gocheck"
+)
+
+type LexerSuite struct {
+}
+
+var _ = Suite(&LexerSuite{})
+
+func (s *LexerSuite) TestLexing(c *C) {
+	_, ch := lex("query", "package (<< 1.3), $Source | !\"app\", 'd\"\\a\\'ta' {i386}")
+
+	c.Check(<-ch, Equals, item{typ: itemString, val: "package"})
+	c.Check(<-ch, Equals, item{typ: itemLeftParen, val: "("})
+	c.Check(<-ch, Equals, item{typ: itemLt, val: "<<"})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "1.3"})
+	c.Check(<-ch, Equals, item{typ: itemRightParen, val: ")"})
+	c.Check(<-ch, Equals, item{typ: itemAnd, val: ","})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "$Source"})
+	c.Check(<-ch, Equals, item{typ: itemOr, val: "|"})
+	c.Check(<-ch, Equals, item{typ: itemNot, val: "!"})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "app"})
+	c.Check(<-ch, Equals, item{typ: itemAnd, val: ","})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "d\"a'ta"})
+	c.Check(<-ch, Equals, item{typ: itemLeftCurly, val: "{"})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "i386"})
+	c.Check(<-ch, Equals, item{typ: itemRightCurly, val: "}"})
+	c.Check(<-ch, Equals, item{typ: itemEOF, val: ""})
+}
+
+func (s *LexerSuite) TestConsume(c *C) {
+	l, _ := lex("query", "package (<< 1.3)")
+
+	c.Check(l.Current(), Equals, item{typ: itemString, val: "package"})
+	c.Check(l.Current(), Equals, item{typ: itemString, val: "package"})
+	l.Consume()
+	c.Check(l.Current(), Equals, item{typ: itemLeftParen, val: "("})
+	l.Consume()
+	c.Check(l.Current(), Equals, item{typ: itemLt, val: "<<"})
+}
+
+func (s *LexerSuite) TestString(c *C) {
+	l, _ := lex("query", "package (<< 1.3)")
+
+	c.Check(fmt.Sprintf("%s", l.Current()), Equals, "\"package\"")
+	l.Consume()
+	c.Check(fmt.Sprintf("%s", l.Current()), Equals, "(")
+}
+
+func (s *LexerSuite) TestError(c *C) {
+	l, _ := lex("query", "'package")
+
+	c.Check(fmt.Sprintf("%s", l.Current()), Equals, "error: unexpected eof in quoted string")
+}

query/query.go

@@ -0,0 +1,29 @@
+// Package query implements query language for
+package query
+
+import (
+	"github.com/smira/aptly/deb"
+)
+
+/*
+
+  Query language resembling Debian dependencies and reprepro
+  queries: http://mirrorer.alioth.debian.org/reprepro.1.html
+
+  Query := A | A '|' Query
+  A := B | B ',' A
+  B := C | '!' B
+  C := '(' Query ')' | D
+  D := <field> <condition> <arch_condition> | <pkg>_<version>_<arch>
+  field := <package-name> | <field> | $special_field
+  condition := '(' <operator> value ')' |
+  arch_condition := '{' arch '}' |
+  operator := | << | < | <= | > | >> | >= | = | % | ~
+*/
+
+// Parse parses input package query into PackageQuery tree ready for evaluation
+func Parse(query string) (result deb.PackageQuery, err error) {
+	l, _ := lex("", query)
+	result, err = parse(l)
+	return
+}

query/query_test.go

@@ -0,0 +1,11 @@
+package query
+
+import (
+	. "launchpad.net/gocheck"
+	"testing"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

query/syntax.go

@@ -0,0 +1,225 @@
+package query
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"regexp"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+type parser struct {
+	name  string // used only for error reports.
+	input *lexer // the input lexer
+	err   error  // error stored while parsing
+}
+
+func parse(input *lexer) (deb.PackageQuery, error) {
+	p := &parser{
+		name:  input.name,
+		input: input,
+	}
+	query := p.parse()
+	if p.err != nil {
+		return nil, p.err
+	}
+	return query, nil
+}
+
+// Entry into parser
+func (p *parser) parse() deb.PackageQuery {
+	defer func() {
+		if r := recover(); r != nil {
+			p.err = fmt.Errorf("parsing failed: %s", r)
+		}
+	}()
+
+	q := p.Query()
+	if p.input.Current().typ != itemEOF {
+		panic(fmt.Sprintf("unexpected token %s: expecting end of query", p.input.Current()))
+	}
+	return q
+}
+
+// Query := A | A '|' Query
+func (p *parser) Query() deb.PackageQuery {
+	q := p.A()
+	if p.input.Current().typ == itemOr {
+		p.input.Consume()
+		return &deb.OrQuery{L: q, R: p.Query()}
+	}
+	return q
+}
+
+// A := B | B ',' A
+func (p *parser) A() deb.PackageQuery {
+	q := p.B()
+	if p.input.Current().typ == itemAnd {
+		p.input.Consume()
+		return &deb.AndQuery{L: q, R: p.A()}
+	}
+	return q
+}
+
+// B := C | '!' B
+func (p *parser) B() deb.PackageQuery {
+	if p.input.Current().typ == itemNot {
+		p.input.Consume()
+		return &deb.NotQuery{Q: p.B()}
+	}
+	return p.C()
+}
+
+// C := '(' Query ')' | D
+func (p *parser) C() deb.PackageQuery {
+	if p.input.Current().typ == itemLeftParen {
+		p.input.Consume()
+		q := p.Query()
+		if p.input.Current().typ != itemRightParen {
+			panic(fmt.Sprintf("unexpected token %s: expecting ')'", p.input.Current()))
+		}
+		p.input.Consume()
+		return q
+	}
+	return p.D()
+}
+
+func operatorToRelation(operator itemType) int {
+	switch operator {
+	case 0:
+		return deb.VersionDontCare
+	case itemLt:
+		return deb.VersionLess
+	case itemLtEq:
+		return deb.VersionLessOrEqual
+	case itemGt:
+		return deb.VersionGreater
+	case itemGtEq:
+		return deb.VersionGreaterOrEqual
+	case itemEq:
+		return deb.VersionEqual
+	case itemPatMatch:
+		return deb.VersionPatternMatch
+	case itemRegexp:
+		return deb.VersionRegexp
+	}
+	panic("unable to map token to relation")
+}
+
+// isPackageRef returns ok true if field has format pkg_version_arch
+func parsePackageRef(query string) (pkg, version, arch string, ok bool) {
+	i := strings.Index(query, "_")
+	if i != -1 {
+		pkg, query = query[:i], query[i+1:]
+		j := strings.LastIndex(query, "_")
+		if j != -1 {
+			version, arch = query[:j], query[j+1:]
+			ok = true
+		}
+	}
+	return
+}
+
+// D := <field> <condition> <arch_condition> | <package>_<version>_<arch>
+// field := <package-name> | <field> | $special_field
+func (p *parser) D() deb.PackageQuery {
+	if p.input.Current().typ != itemString {
+		panic(fmt.Sprintf("unexpected token %s: expecting field or package name", p.input.Current()))
+	}
+
+	field := p.input.Current().val
+	p.input.Consume()
+
+	operator, value := p.Condition()
+
+	r, _ := utf8.DecodeRuneInString(field)
+	if strings.HasPrefix(field, "$") || unicode.IsUpper(r) {
+		// special field or regular field
+		q := &deb.FieldQuery{Field: field, Relation: operatorToRelation(operator), Value: value}
+		if q.Relation == deb.VersionRegexp {
+			var err error
+			q.Regexp, err = regexp.Compile(q.Value)
+			if err != nil {
+				panic(fmt.Sprintf("regexp compile failed: %s", err))
+			}
+		}
+		return q
+	} else if operator == 0 && value == "" {
+		if pkg, version, arch, ok := parsePackageRef(field); ok {
+			// query for specific package
+			return &deb.PkgQuery{Pkg: pkg, Version: version, Arch: arch}
+		}
+	}
+
+	// regular dependency-like query
+	q := &deb.DependencyQuery{Dep: deb.Dependency{
+		Pkg:          field,
+		Relation:     operatorToRelation(operator),
+		Version:      value,
+		Architecture: p.ArchCondition()}}
+	if q.Dep.Relation == deb.VersionRegexp {
+		var err error
+		q.Dep.Regexp, err = regexp.Compile(q.Dep.Version)
+		if err != nil {
+			panic(fmt.Sprintf("regexp compile failed: %s", err))
+		}
+	}
+	return q
+}
+
+// condition := '(' <operator> value ')' |
+// operator := | << | < | <= | > | >> | >= | = | % | ~
+func (p *parser) Condition() (operator itemType, value string) {
+	if p.input.Current().typ != itemLeftParen {
+		return
+	}
+	p.input.Consume()
+
+	if p.input.Current().typ == itemLt ||
+		p.input.Current().typ == itemGt ||
+		p.input.Current().typ == itemLtEq ||
+		p.input.Current().typ == itemGtEq ||
+		p.input.Current().typ == itemEq ||
+		p.input.Current().typ == itemPatMatch ||
+		p.input.Current().typ == itemRegexp {
+		operator = p.input.Current().typ
+		p.input.Consume()
+	} else {
+		operator = itemEq
+	}
+
+	if p.input.Current().typ != itemString {
+		panic(fmt.Sprintf("unexpected token %s: expecting value", p.input.Current()))
+	}
+	value = p.input.Current().val
+	p.input.Consume()
+
+	if p.input.Current().typ != itemRightParen {
+		panic(fmt.Sprintf("unexpected token %s: expecting ')'", p.input.Current()))
+	}
+	p.input.Consume()
+
+	return
+}
+
+// arch_condition := '{' arch '}' |
+func (p *parser) ArchCondition() (arch string) {
+	if p.input.Current().typ != itemLeftCurly {
+		return
+	}
+	p.input.Consume()
+
+	if p.input.Current().typ != itemString {
+		panic(fmt.Sprintf("unexpected token %s: expecting architecture", p.input.Current()))
+	}
+	arch = p.input.Current().val
+	p.input.Consume()
+
+	if p.input.Current().typ != itemRightCurly {
+		panic(fmt.Sprintf("unexpected token %s: expecting '}'", p.input.Current()))
+	}
+	p.input.Consume()
+
+	return
+}

query/syntax_test.go

@@ -0,0 +1,98 @@
+package query
+
+import (
+	"github.com/smira/aptly/deb"
+	. "launchpad.net/gocheck"
+	"regexp"
+)
+
+type SyntaxSuite struct {
+}
+
+var _ = Suite(&SyntaxSuite{})
+
+func (s *SyntaxSuite) TestParsing(c *C) {
+	l, _ := lex("query", "package (<< 1.3~dev), $Source")
+	q, err := parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q.(*deb.AndQuery).L, DeepEquals, &deb.DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionLess, Version: "1.3~dev"}})
+	c.Check(q.(*deb.AndQuery).R, DeepEquals, &deb.FieldQuery{Field: "$Source"})
+
+	l, _ = lex("query", "package (1.3), Name (lala) | !$Source")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q.(*deb.OrQuery).L.(*deb.AndQuery).L, DeepEquals, &deb.DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionEqual, Version: "1.3"}})
+	c.Check(q.(*deb.OrQuery).L.(*deb.AndQuery).R, DeepEquals, &deb.FieldQuery{Field: "Name", Relation: deb.VersionEqual, Value: "lala"})
+	c.Check(q.(*deb.OrQuery).R.(*deb.NotQuery).Q, DeepEquals, &deb.FieldQuery{Field: "$Source"})
+
+	l, _ = lex("query", "package, ((!(Name | $Source (~ a.*))))")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q.(*deb.AndQuery).L, DeepEquals, &deb.DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionDontCare}})
+	c.Check(q.(*deb.AndQuery).R.(*deb.NotQuery).Q.(*deb.OrQuery).L, DeepEquals, &deb.FieldQuery{Field: "Name", Relation: deb.VersionDontCare})
+	c.Check(q.(*deb.AndQuery).R.(*deb.NotQuery).Q.(*deb.OrQuery).R, DeepEquals, &deb.FieldQuery{Field: "$Source", Relation: deb.VersionRegexp, Value: "a.*",
+		Regexp: regexp.MustCompile("a.*")})
+
+	l, _ = lex("query", "package (> 5.3.7)")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q, DeepEquals, &deb.DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionGreaterOrEqual, Version: "5.3.7"}})
+
+	l, _ = lex("query", "package (~ 5\\.3.*~dev)")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q, DeepEquals, &deb.DependencyQuery{Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionRegexp, Version: "5\\.3.*~dev",
+		Regexp: regexp.MustCompile("5\\.3.*~dev")}})
+
+	l, _ = lex("query", "alien-data_1.3.4~dev_i386")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q, DeepEquals, &deb.PkgQuery{Pkg: "alien-data", Version: "1.3.4~dev", Arch: "i386"})
+
+	l, _ = lex("query", "package (> 5.3.7) {amd64}")
+	q, err = parse(l)
+
+	c.Assert(err, IsNil)
+	c.Check(q, DeepEquals, &deb.DependencyQuery{
+		Dep: deb.Dependency{Pkg: "package", Relation: deb.VersionGreaterOrEqual, Version: "5.3.7", Architecture: "amd64"}})
+}
+
+func (s *SyntaxSuite) TestParsingErrors(c *C) {
+	l, _ := lex("query", "package (> 5.3.7), ")
+	_, err := parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: unexpected token <EOL>: expecting field or package name")
+
+	l, _ = lex("query", "package>5.3.7)")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: unexpected token \\): expecting end of query")
+
+	l, _ = lex("query", "package | !|")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: unexpected token |: expecting field or package name")
+
+	l, _ = lex("query", "((package )")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: unexpected token <EOL>: expecting '\\)'")
+
+	l, _ = lex("query", "!package )")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: unexpected token \\): expecting end of query")
+
+	l, _ = lex("query", "'package )")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: unexpected token error: unexpected eof in quoted string: expecting field or package name")
+
+	l, _ = lex("query", "package (~ 1.2[34)")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: regexp compile failed: error parsing regexp: missing closing \\]: `\\[34`")
+
+	l, _ = lex("query", "$Name (~ 1.2[34)")
+	_, err = parse(l)
+	c.Check(err, ErrorMatches, "parsing failed: regexp compile failed: error parsing regexp: missing closing \\]: `\\[34`")
+}

s3/public.go

@@ -0,0 +1,253 @@
+package s3
+
+import (
+	"fmt"
+	"github.com/mitchellh/goamz/aws"
+	"github.com/mitchellh/goamz/s3"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/files"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// PublishedStorage abstract file system with published files (actually hosted on S3)
+type PublishedStorage struct {
+	s3               *s3.S3
+	bucket           *s3.Bucket
+	acl              s3.ACL
+	prefix           string
+	storageClass     string
+	encryptionMethod string
+	plusWorkaround   bool
+}
+
+// Check interface
+var (
+	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+)
+
+// NewPublishedStorageRaw creates published storage from raw aws credentials
+func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL, prefix,
+	storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
+	if defaultACL == "" {
+		defaultACL = "private"
+	}
+
+	if storageClass == "STANDARD" {
+		storageClass = ""
+	}
+
+	result := &PublishedStorage{
+		s3:               s3.New(auth, region),
+		acl:              s3.ACL(defaultACL),
+		prefix:           prefix,
+		storageClass:     storageClass,
+		encryptionMethod: encryptionMethod,
+		plusWorkaround:   plusWorkaround}
+	result.bucket = result.s3.Bucket(bucket)
+
+	return result, nil
+}
+
+// NewPublishedStorage creates new instance of PublishedStorage with specified S3 access
+// keys, region and bucket name
+func NewPublishedStorage(accessKey, secretKey, region, bucket, defaultACL, prefix,
+	storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
+	auth, err := aws.GetAuth(accessKey, secretKey)
+	if err != nil {
+		return nil, err
+	}
+
+	awsRegion, ok := aws.Regions[region]
+	if !ok {
+		return nil, fmt.Errorf("unknown region: %#v", region)
+	}
+
+	return NewPublishedStorageRaw(auth, awsRegion, bucket, defaultACL, prefix, storageClass, encryptionMethod, plusWorkaround)
+}
+
+// String
+func (storage *PublishedStorage) String() string {
+	return fmt.Sprintf("S3: %s:%s/%s", storage.s3.Region.Name, storage.bucket.Name, storage.prefix)
+}
+
+// MkDir creates directory recursively under public path
+func (storage *PublishedStorage) MkDir(path string) error {
+	// no op for S3
+	return nil
+}
+
+// PutFile puts file into published storage at specified path
+func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
+	var (
+		source *os.File
+		err    error
+		fi     os.FileInfo
+	)
+	source, err = os.Open(sourceFilename)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	fi, err = source.Stat()
+	if err != nil {
+		return err
+	}
+
+	headers := map[string][]string{
+		"Content-Type": {"binary/octet-stream"},
+	}
+	if storage.storageClass != "" {
+		headers["x-amz-storage-class"] = []string{storage.storageClass}
+	}
+	if storage.encryptionMethod != "" {
+		headers["x-amz-server-side-encryption"] = []string{storage.encryptionMethod}
+	}
+
+	err = storage.bucket.PutReaderHeader(filepath.Join(storage.prefix, path), source, fi.Size(), headers, storage.acl)
+	if err != nil {
+		return fmt.Errorf("error uploading %s to %s: %s", sourceFilename, storage, err)
+	}
+
+	if storage.plusWorkaround && strings.Index(path, "+") != -1 {
+		return storage.PutFile(strings.Replace(path, "+", " ", -1), sourceFilename)
+	}
+	return nil
+}
+
+// Remove removes single file under public path
+func (storage *PublishedStorage) Remove(path string) error {
+	err := storage.bucket.Del(filepath.Join(storage.prefix, path))
+	if err != nil {
+		return fmt.Errorf("error deleting %s from %s: %s", path, storage, err)
+	}
+	return nil
+}
+
+// RemoveDirs removes directory structure under public path
+func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
+	const page = 1000
+
+	filelist, err := storage.Filelist(path)
+	if err != nil {
+		return err
+	}
+
+	numParts := (len(filelist) + page - 1) / page
+
+	for i := 0; i < numParts; i++ {
+		var part []string
+		if i == numParts-1 {
+			part = filelist[i*page:]
+		} else {
+			part = filelist[i*page : (i+1)*page]
+		}
+		paths := make([]string, len(part))
+
+		for i := range part {
+			paths[i] = filepath.Join(storage.prefix, path, part[i])
+		}
+
+		err = storage.bucket.MultiDel(paths)
+		if err != nil {
+			return fmt.Errorf("error deleting multiple paths from %s: %s", storage, err)
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// LinkFromPool links package file from pool to dist's pool location
+//
+// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
+// sourcePool is instance of aptly.PackagePool
+// sourcePath is filepath to package file in package pool
+//
+// LinkFromPool returns relative path for the published file to be included in package index
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool,
+	sourcePath, sourceMD5 string, force bool) error {
+	// verify that package pool is local pool in filesystem
+	_ = sourcePool.(*files.PackagePool)
+
+	baseName := filepath.Base(sourcePath)
+	relPath := filepath.Join(publishedDirectory, baseName)
+	poolPath := filepath.Join(storage.prefix, relPath)
+
+	var (
+		dstKey *s3.Key
+		err    error
+	)
+
+	dstKey, err = storage.bucket.GetKey(poolPath)
+	if err != nil {
+		if s3err, ok := err.(*s3.Error); !ok || s3err.StatusCode != 404 {
+			return fmt.Errorf("error getting information about %s from %s: %s", poolPath, storage, err)
+		}
+	} else {
+		destinationMD5 := strings.Replace(dstKey.ETag, "\"", "", -1)
+		if destinationMD5 == sourceMD5 {
+			return nil
+		}
+
+		if !force && destinationMD5 != sourceMD5 {
+			return fmt.Errorf("error putting file to %s: file already exists and is different: %s", poolPath, storage)
+
+		}
+	}
+
+	return storage.PutFile(relPath, sourcePath)
+}
+
+// Filelist returns list of files under prefix
+func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
+	result := []string{}
+	marker := ""
+	prefix = filepath.Join(storage.prefix, prefix)
+	if prefix != "" {
+		prefix += "/"
+	}
+	for {
+		contents, err := storage.bucket.List(prefix, "", marker, 1000)
+		if err != nil {
+			return nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
+		}
+		lastKey := ""
+		for _, key := range contents.Contents {
+			if prefix == "" {
+				result = append(result, key.Key)
+			} else {
+				result = append(result, key.Key[len(prefix):])
+			}
+			lastKey = key.Key
+		}
+		if contents.IsTruncated {
+			marker = contents.NextMarker
+			if marker == "" {
+				// From the s3 docs: If response does not include the
+				// NextMarker and it is truncated, you can use the value of the
+				// last Key in the response as the marker in the subsequent
+				// request to get the next set of object keys.
+				marker = lastKey
+			}
+		} else {
+			break
+		}
+	}
+
+	return result, nil
+}
+
+// RenameFile renames (moves) file
+func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
+	err := storage.bucket.Copy(filepath.Join(storage.prefix, oldName), filepath.Join(storage.prefix, newName), storage.acl)
+	if err != nil {
+		return fmt.Errorf("error copying %s -> %s in %s: %s", oldName, newName, storage, err)
+	}
+
+	return storage.Remove(oldName)
+}

s3/public_test.go

@@ -0,0 +1,192 @@
+package s3
+
+import (
+	"github.com/mitchellh/goamz/aws"
+	"github.com/mitchellh/goamz/s3/s3test"
+	"github.com/smira/aptly/files"
+	"io/ioutil"
+	. "launchpad.net/gocheck"
+	"os"
+	"path/filepath"
+)
+
+type PublishedStorageSuite struct {
+	srv                      *s3test.Server
+	storage, prefixedStorage *PublishedStorage
+}
+
+var _ = Suite(&PublishedStorageSuite{})
+
+func (s *PublishedStorageSuite) SetUpTest(c *C) {
+	var err error
+	s.srv, err = s3test.NewServer(&s3test.Config{})
+	c.Assert(err, IsNil)
+	c.Assert(s.srv, NotNil)
+
+	auth, _ := aws.GetAuth("aa", "bb")
+	s.storage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "", "", "", false)
+	c.Assert(err, IsNil)
+
+	s.prefixedStorage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "lala", "", "", false)
+	c.Assert(err, IsNil)
+
+	err = s.storage.s3.Bucket("test").PutBucket("private")
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) TearDownTest(c *C) {
+	s.srv.Quit()
+}
+
+func (s *PublishedStorageSuite) TestNewPublishedStorage(c *C) {
+	stor, err := NewPublishedStorage("aa", "bbb", "", "", "", "", "", "", false)
+	c.Check(stor, IsNil)
+	c.Check(err, ErrorMatches, "unknown region: .*")
+}
+
+func (s *PublishedStorageSuite) TestPutFile(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to s3!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("a/b.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	data, err := s.storage.bucket.Get("a/b.txt")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+
+	err = s.prefixedStorage.PutFile("a/b.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	data, err = s.storage.bucket.Get("lala/a/b.txt")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+}
+
+func (s *PublishedStorageSuite) TestPutFilePlusWorkaround(c *C) {
+	s.storage.plusWorkaround = true
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to s3!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("a/b+c.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	data, err := s.storage.bucket.Get("a/b+c.txt")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+
+	data, err = s.storage.bucket.Get("a/b c.txt")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+}
+
+func (s *PublishedStorageSuite) TestFilelist(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")
+		c.Check(err, IsNil)
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestRemove(c *C) {
+	err := s.storage.bucket.Put("a/b", []byte("test"), "binary/octet-stream", "private")
+	c.Check(err, IsNil)
+
+	err = s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	_, err = s.storage.bucket.Get("a/b")
+	c.Check(err, ErrorMatches, "The specified key does not exist.")
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
+	c.Skip("multiple-delete not available in s3test")
+
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		err := s.storage.bucket.Put(path, []byte("test"), "binary/octet-stream", "private")
+		c.Check(err, IsNil)
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
+
+}
+
+func (s *PublishedStorageSuite) TestRenameFile(c *C) {
+	c.Skip("copy not available in s3test")
+}
+
+func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
+	root := c.MkDir()
+	pool := files.NewPackagePool(root)
+
+	sourcePath := filepath.Join(root, "pool/c1/df/mars-invaders_1.03.deb")
+	err := os.MkdirAll(filepath.Dir(sourcePath), 0755)
+	c.Assert(err, IsNil)
+
+	err = ioutil.WriteFile(sourcePath, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+
+	sourcePath2 := filepath.Join(root, "pool/e9/df/mars-invaders_1.03.deb")
+	err = os.MkdirAll(filepath.Dir(sourcePath2), 0755)
+	c.Assert(err, IsNil)
+
+	err = ioutil.WriteFile(sourcePath2, []byte("Spam"), 0644)
+	c.Assert(err, IsNil)
+
+	// first link from pool
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "c1df1da7a1ce305a3b60af9d5733ac1d", false)
+	c.Check(err, IsNil)
+
+	data, err := s.storage.bucket.Get("pool/main/m/mars-invaders/mars-invaders_1.03.deb")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("Contents"))
+
+	// duplicate link from pool
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "c1df1da7a1ce305a3b60af9d5733ac1d", false)
+	c.Check(err, IsNil)
+
+	data, err = s.storage.bucket.Get("pool/main/m/mars-invaders/mars-invaders_1.03.deb")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath2, "e9dfd31cc505d51fc26975250750deab", false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
+
+	data, err = s.storage.bucket.Get("pool/main/m/mars-invaders/mars-invaders_1.03.deb")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict and force
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath2, "e9dfd31cc505d51fc26975250750deab", true)
+	c.Check(err, IsNil)
+
+	data, err = s.storage.bucket.Get("pool/main/m/mars-invaders/mars-invaders_1.03.deb")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("Spam"))
+}

s3/s3.go

@@ -0,0 +1,2 @@
+// Package s3 handles publishing to Amazon S3
+package s3

s3/s3_test.go

@@ -0,0 +1,11 @@
+package s3
+
+import (
+	. "launchpad.net/gocheck"
+	"testing"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

system/fixture.sh

@@ -27,3 +27,6 @@ aptly mirror update gnuplot-maverick
 
 aptly mirror create -with-sources gnuplot-maverick-src http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick
 aptly mirror update gnuplot-maverick-src
+
+aptly mirror create sensu http://repos.sensuapp.org/apt sensu
+aptly mirror update sensu

system/lib.py

@@ -66,6 +66,7 @@ class BaseTest(object):
     configFile = {
         "rootDir": "%s/.aptly" % os.environ["HOME"],
         "downloadConcurrency": 4,
+        "downloadSpeedLimit": 0,
         "architectures": [],
         "dependencyFollowSuggests": False,
         "dependencyFollowRecommends": False,
@@ -84,6 +85,8 @@ class BaseTest(object):
 
     outputMatchPrepare = None
 
+    captureResults = False
+
     def test(self):
         self.prepare()
         self.run()
@@ -152,6 +155,7 @@ class BaseTest(object):
             if not hasattr(command, "__iter__"):
                 params = {
                     'files': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files"),
+                    'udebs': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "udebs"),
                     'testfiles': os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), self.__class__.__name__),
                 }
                 if self.fixtureWebServer:
@@ -181,15 +185,36 @@ class BaseTest(object):
     def expand_environ(self, gold):
         return string.Template(gold).substitute(os.environ)
 
+    def get_gold_filename(self, gold_name="gold"):
+        return os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), self.__class__.__name__ + "_" + gold_name)
+
     def get_gold(self, gold_name="gold"):
-        gold = os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), self.__class__.__name__ + "_" + gold_name)
-        return self.gold_processor(open(gold, "r").read())
+        return self.gold_processor(open(self.get_gold_filename(gold_name), "r").read())
 
     def check_output(self):
-        self.verify_match(self.get_gold(), self.output, match_prepare=self.outputMatchPrepare)
+        try:
+            self.verify_match(self.get_gold(), self.output, match_prepare=self.outputMatchPrepare)
+        except:
+            if self.captureResults:
+                if self.outputMatchPrepare is not None:
+                    self.output = self.outputMatchPrepare(self.output)
+                with open(self.get_gold_filename(), "w") as f:
+                    f.write(self.output)
+            else:
+                raise
 
     def check_cmd_output(self, command, gold_name, match_prepare=None, expected_code=0):
-        self.verify_match(self.get_gold(gold_name), self.run_cmd(command, expected_code=expected_code), match_prepare)
+        output = self.run_cmd(command, expected_code=expected_code)
+        try:
+            self.verify_match(self.get_gold(gold_name), output, match_prepare)
+        except:
+            if self.captureResults:
+                if match_prepare is not None:
+                    output = match_prepare(output)
+                with open(self.get_gold_filename(gold_name), "w") as f:
+                    f.write(output)
+            else:
+                raise
 
     def read_file(self, path):
         with open(os.path.join(os.environ["HOME"], ".aptly", path), "r") as f:
@@ -200,11 +225,26 @@ class BaseTest(object):
 
     def check_file_contents(self, path, gold_name, match_prepare=None):
         contents = self.read_file(path)
+        try:
 
-        self.verify_match(self.get_gold(gold_name), contents, match_prepare=match_prepare)
+            self.verify_match(self.get_gold(gold_name), contents, match_prepare=match_prepare)
+        except:
+            if self.captureResults:
+                with open(self.get_gold_filename(gold_name), "w") as f:
+                    f.write(contents)
+            else:
+                raise
 
     def check_file(self):
-        self.verify_match(self.get_gold(), open(self.checkedFile, "r").read())
+        contents = open(self.checkedFile, "r").read()
+        try:
+            self.verify_match(self.get_gold(), contents)
+        except:
+            if self.captureResults:
+                with open(self.get_gold_filename(), "w") as f:
+                    f.write(contents)
+            else:
+                raise
 
     def check_exists(self, path):
         if not os.path.exists(os.path.join(os.environ["HOME"], ".aptly", path)):

system/run.py

@@ -4,9 +4,11 @@ import glob
 import importlib
 import os
 import inspect
+import fnmatch
 import sys
 
 from lib import BaseTest
+from s3_lib import S3Test
 
 try:
     from termcolor import colored
@@ -15,11 +17,11 @@ except ImportError:
         return s
 
 
-def run(include_long_tests=False, tests=None):
+def run(include_long_tests=False, capture_results=False, tests=None, filters=None):
     """
     Run system test.
     """
-    if tests is None:
+    if not tests:
         tests = glob.glob("t*_*")
     fails = []
     numTests = numFailed = numSkipped = 0
@@ -31,9 +33,21 @@ def run(include_long_tests=False, tests=None):
         for name in dir(testModule):
             o = getattr(testModule, name)
 
-            if not (inspect.isclass(o) and issubclass(o, BaseTest) and o is not BaseTest):
+            if not (inspect.isclass(o) and issubclass(o, BaseTest) and o is not BaseTest and
+                    o is not S3Test):
                 continue
 
+            if filters:
+                matches = False
+
+                for filt in filters:
+                    if fnmatch.fnmatch(o.__name__, filt):
+                        matches = True
+                        break
+
+                if not matches:
+                    continue
+
             t = o()
             if t.longTest and not include_long_tests or not t.fixture_available():
                 numSkipped += 1
@@ -44,6 +58,7 @@ def run(include_long_tests=False, tests=None):
             sys.stdout.write("%s:%s... " % (test, o.__name__))
 
             try:
+                t.captureResults = capture_results
                 t.test()
             except BaseException, e:
                 numFailed += 1
@@ -69,10 +84,25 @@ def run(include_long_tests=False, tests=None):
 if __name__ == "__main__":
     os.chdir(os.path.realpath(os.path.dirname(sys.argv[0])))
     include_long_tests = False
+    capture_results = False
     tests = None
-    if len(sys.argv) > 1:
-        if sys.argv[1] == "--long":
+    args = sys.argv[1:]
+
+    while len(args) > 0 and args[0].startswith("--"):
+        if args[0] == "--long":
             include_long_tests = True
+        elif args[0] == "--capture":
+            capture_results = True
+
+        args = args[1:]
+
+    tests = []
+    filters = []
+
+    for arg in args:
+        if arg.startswith('t'):
+            tests.append(arg)
         else:
-            tests = sys.argv[1:]
-    run(include_long_tests, tests)
+            filters.append(arg)
+
+    run(include_long_tests, capture_results, tests, filters)

system/s3_lib.py

@@ -0,0 +1,78 @@
+from lib import BaseTest
+import uuid
+import os
+
+try:
+    import boto
+
+    if 'AWS_SECRET_ACCESS_KEY' in os.environ and 'AWS_ACCESS_KEY_ID' in os.environ:
+        s3_conn = boto.connect_s3()
+    else:
+        s3_conn = None
+except ImportError:
+    s3_conn = None
+
+
+class S3Test(BaseTest):
+    """
+    BaseTest + support for S3
+    """
+
+    def fixture_available(self):
+        return super(S3Test, self).fixture_available() and s3_conn is not None
+
+    def prepare(self):
+        self.bucket_name = "aptly-sys-test-" + str(uuid.uuid4())
+        self.bucket = s3_conn.create_bucket(self.bucket_name)
+        self.configOverride["S3PublishEndpoints"] = {
+            "test1": {
+                "region": "us-east-1",
+                "bucket": self.bucket_name,
+            }
+        }
+
+        super(S3Test, self).prepare()
+
+    def shutdown(self):
+        if hasattr(self, "bucket_name"):
+            if hasattr(self, "bucket"):
+                keys = self.bucket.list()
+                if keys:
+                    self.bucket.delete_keys(keys)
+            s3_conn.delete_bucket(self.bucket_name)
+
+        super(S3Test, self).shutdown()
+
+    def check_path(self, path):
+        if not hasattr(self, "bucket_contents"):
+            self.bucket_contents = [key.name for key in self.bucket.list()]
+
+        if path.startswith("public/"):
+            path = path[7:]
+
+        if path in self.bucket_contents:
+            return True
+
+        if not path.endswith("/"):
+            path = path + "/"
+
+        for item in self.bucket_contents:
+            if item.startswith(path):
+                return True
+
+        return False
+
+    def check_exists(self, path):
+        if not self.check_path(path):
+            raise Exception("path %s doesn't exist" % (path, ))
+
+    def check_not_exists(self, path):
+        if self.check_path(path):
+            raise Exception("path %s exists" % (path, ))
+
+    def read_file(self, path):
+        if path.startswith("public/"):
+            path = path[7:]
+
+        key = self.bucket.get_key(path)
+        return key.get_contents_as_string()

system/t01_version/VersionTest_gold

@@ -1 +1 @@
-aptly version: 0.6
+aptly version: 0.8

system/t02_config/CreateConfigTest_gold

@@ -1,6 +1,7 @@
 {
   "rootDir": "${HOME}/.aptly",
   "downloadConcurrency": 4,
+  "downloadSpeedLimit": 0,
   "architectures": [],
   "dependencyFollowSuggests": false,
   "dependencyFollowRecommends": false,
@@ -10,5 +11,6 @@
   "gpgDisableVerify": false,
   "downloadSourcePackages": false,
   "ppaDistributorID": "ubuntu",
-  "ppaCodename": ""
+  "ppaCodename": "",
+  "S3PublishEndpoints": {}
 }

system/t03_help/MainHelpTest_gold

@@ -4,9 +4,9 @@ upgrade individual packages, take snapshots and publish them
 back as Debian repositories.
 
 aptly's goal is to establish repeatability and controlled changes
-in a package-centric environment. aptly allows to fix a set of packages
+in a package-centric environment. aptly allows one to fix a set of packages
 in a repository, so that package installation and upgrade becomes
-deterministic. At the same time aptly allows to perform controlled,
+deterministic. At the same time aptly allows one to perform controlled,
 fine-grained changes in repository contents to transition your
 package environment to new version.
 

system/t03_help/MainTest_gold

@@ -5,6 +5,7 @@ Commands:
     db          manage aptly's internal database and package pool
     graph       render graph of relationships
     mirror      manage mirrors of remote repositories
+    package     operations on packages
     publish     manage published repositories
     repo        manage local package repositories
     serve       HTTP serve published repositories

system/t03_help/MirrorCreateHelpTest_gold

@@ -1,7 +1,7 @@
 Usage: aptly mirror create <name> <archive url> <distribution> [<component1> ...]
 
 Creates mirror <name> of remote repository, aptly supports both regular and flat Debian repositories exported
-via HTTP. aptly would try download Release file from remote repository and verify its' signature. Command
+via HTTP and FTP. aptly would try download Release file from remote repository and verify its' signature. Command
 line format resembles apt utlitily sources.list(5).
 
 PPA urls could specified in short format:
@@ -19,7 +19,10 @@ Options:
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests
+  -filter="": filter packages in mirror
+  -filter-with-deps=false: when filtering, include dependencies of matching packages as well
   -ignore-signatures=false: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
   -with-sources=false: download source packages in addition to binary packages
+  -with-udebs=false: download .udeb packages (Debian installer support)
 

system/t03_help/MirrorCreateTest_gold

@@ -10,7 +10,10 @@ Options:
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests
+  -filter="": filter packages in mirror
+  -filter-with-deps=false: when filtering, include dependencies of matching packages as well
   -ignore-signatures=false: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
   -with-sources=false: download source packages in addition to binary packages
+  -with-udebs=false: download .udeb packages (Debian installer support)
 ERROR: unable to parse command

system/t03_help/MirrorHelpTest_gold

@@ -4,7 +4,10 @@ Commands:
 
     create      create new mirror
     drop        delete mirror
+    edit        edit mirror settings
     list        list mirrors
+    rename      renames mirror
+    search      search mirror for packages matching query
     show        show details about mirror
     update      update mirror
 

system/t03_help/MirrorTest_gold

@@ -4,7 +4,10 @@ Commands:
 
     create      create new mirror
     drop        delete mirror
+    edit        edit mirror settings
     list        list mirrors
+    rename      renames mirror
+    search      search mirror for packages matching query
     show        show details about mirror
     update      update mirror
 

system/t03_help/WrongFlagTest_gold

@@ -11,7 +11,10 @@ Options:
   -dep-follow-recommends=false: when processing dependencies, follow Recommends
   -dep-follow-source=false: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests=false: when processing dependencies, follow Suggests
+  -filter="": filter packages in mirror
+  -filter-with-deps=false: when filtering, include dependencies of matching packages as well
   -ignore-signatures=false: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
   -with-sources=false: download source packages in addition to binary packages
+  -with-udebs=false: download .udeb packages (Debian installer support)
 ERROR: unable to parse flags