aptly version 0.7

as sort key.

.travis.yml

@@ -2,8 +2,8 @@ language: go
 
 go:
   - 1.1
-  - 1.2
   - 1.2.1
+  - 1.3
   - tip
 
 env:

AUTHORS

@@ -0,0 +1,6 @@
+List of contributors, in chronological order:
+
+* Andrey Smirnov (https://github.com/smira)
+* Sebastien Binet (https://github.com/sbinet)
+* Ryan Uber (https://github.com/ryanuber)
+* Simon Aquino (https://github.com/simonaquino)

Gomfile

@@ -1,12 +1,14 @@
 gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
 gom 'code.google.com/p/go.crypto/ssh/terminal', :commit => '7aa593ce8cea'
-gom 'code.google.com/p/gographviz', :commit => '212766062629'
+gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2'
+gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556'
 gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
 gom 'github.com/cheggaaa/pb', :commit => '74be7a1388046f374ac36e93d46f5d56e856f827'
-gom 'github.com/gonuts/commander', :commit => 'f8ba4e959ca914268227c3ebbd7f6bf0bb35541a'
-gom 'github.com/gonuts/flag', :commit => '741a6cbd37a30dedc93f817e7de6aaf0ca38a493'
+gom 'github.com/mitchellh/goamz/s3', :commit => '55f224c07975fddef9d2116600c664e30df3d594'
 gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => '527a7b286bd095794af6c519627b7ed3d8fd067a'
+gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
+gom 'github.com/smira/flag', :commit => '0d0aac2addb39050f45e92c5a6252926096dc841'
+gom 'github.com/syndtr/goleveldb/leveldb', :commit => '9888007'
 gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
 gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
 

Makefile

@@ -1,6 +1,6 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-PACKAGES=database debian files http utils
-ALL_PACKAGES=aptly cmd console database debian files http utils
+PACKAGES=database deb files http query s3 utils
+ALL_PACKAGES=aptly cmd console database deb files http query s3 utils
 BINPATH=$(abspath ./_vendor/bin)
 GOM_ENVIRONMENT=-test
 PYTHON?=python
@@ -43,7 +43,7 @@ install:
 	$(GOM) build -o $(BINPATH)/aptly
 
 system-test: install
-ifeq ($(GOVERSION),$(filter $(GOVERSION),go1.2 go1.2.1 devel))
+ifeq ($(GOVERSION),$(filter $(GOVERSION),go1.2 go1.2.1 go1.3 devel))
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 endif
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
@@ -61,4 +61,25 @@ mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-.PHONY: coverage.out
+package:
+	rm -rf root/
+	mkdir -p root/usr/bin/ root/usr/share/man/man1/ root/etc/bash_completion.d
+	cp $(BINPATH)/aptly root/usr/bin
+	cp man/aptly.1 root/usr/share/man/man1
+	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
+	gzip root/usr/share/man/man1/aptly.1
+	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
+	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 -C root/ .
+	mv aptly_$(VERSION)_*.deb ~
+
+src-package:
+	rm -rf aptly-$(VERSION)
+	mkdir -p aptly-$(VERSION)/src/github.com/smira/aptly/
+	cd aptly-$(VERSION)/src/github.com/smira/ && git clone https://github.com/smira/aptly && cd aptly && git checkout v$(VERSION)
+	cd aptly-$(VERSION)/src/github.com/smira/aptly && gom -production install
+	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . \( -name .git -o -name .bzr -o -name .hg \) -print | xargs rm -rf
+	rm -rf aptly-$(VERSION)/src/github.com/smira/aptly/_vendor/{pkg,bin}
+	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
+	rm -rf aptly-$(VERSION)
+
+.PHONY: coverage.out

README.rst

@@ -32,6 +32,23 @@ Current limitations:
 Download
 --------
 
+To install aptly on Debian/Ubuntu, add new repository to /etc/apt/sources.list::
+
+    deb http://repo.aptly.info/ squeeze main
+
+And import key that is used to sign the release::
+
+    $ gpg --keyserver keys.gnupg.net --recv-keys 2A194991
+    $ gpg -a --export 2A194991 | sudo apt-key add -
+
+After that you can install aptly as any other software package::
+
+    $ apt-get update
+    $ apt-get install aptly
+
+Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
+Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
+
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
 If you have Go environment set up, you can build aptly from source by running (go 1.1+ required)::
@@ -40,7 +57,7 @@ If you have Go environment set up, you can build aptly from source by running (g
     mkdir -p $GOPATH/src/github.com/smira/aptly
     git clone https://github.com/smira/aptly $GOPATH/src/github.com/smira/aptly
     cd $GOPATH/src/github.com/smira/aptly
-    gom install
+    gom -production install
     gom build -o $GOPATH/bin/aptly
 
 Aptly is using `gom <https://github.com/mattn/gom>`_ to fix external dependencies, so regular ``go get github.com/smira/aptly``

aptly/interfaces.go

@@ -5,7 +5,6 @@ package aptly
 import (
 	"github.com/smira/aptly/utils"
 	"io"
-	"os"
 )
 
 // PackagePool is asbtraction of package pool storage.
@@ -26,18 +25,32 @@ type PackagePool interface {
 
 // PublishedStorage is abstraction of filesystem storing all published repositories
 type PublishedStorage interface {
-	// PublicPath returns root of public part
-	PublicPath() string
 	// MkDir creates directory recursively under public path
 	MkDir(path string) error
-	// CreateFile creates file for writing under public path
-	CreateFile(path string) (*os.File, error)
+	// PutFile puts file into published storage at specified path
+	PutFile(path string, sourceFilename string) error
 	// RemoveDirs removes directory structure under public path
-	RemoveDirs(path string) error
+	RemoveDirs(path string, progress Progress) error
+	// Remove removes single file under public path
+	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(prefix string, component string, poolDirectory string, sourcePool PackagePool, sourcePath string) (string, error)
-	// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
-	ChecksumsForFile(path string) (utils.ChecksumInfo, error)
+	LinkFromPool(publishedDirectory string, sourcePool PackagePool, sourcePath, sourceMD5 string) error
+	// Filelist returns list of files under prefix
+	Filelist(prefix string) ([]string, error)
+	// RenameFile renames (moves) file
+	RenameFile(oldName, newName string) error
+}
+
+// LocalPublishedStorage is published storage on local filesystem
+type LocalPublishedStorage interface {
+	// PublicPath returns root of public part
+	PublicPath() string
+}
+
+// PublishedStorageProvider is a thing that returns PublishedStorage by name
+type PublishedStorageProvider interface {
+	// GetPublishedStorage returns PublishedStorage by name
+	GetPublishedStorage(name string) PublishedStorage
 }
 
 // Progress is a progress displaying entity, it allows progress bars & simple prints

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
 // Version of aptly
-const Version = "0.4.1"
+const Version = "0.7"
 
 // Enable debugging features?
 const EnableDebug = false

cmd/cmd.go

@@ -3,28 +3,26 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
 	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"os"
 	"time"
 )
 
 // ListPackagesRefList shows list of packages in PackageRefList
-func ListPackagesRefList(reflist *debian.PackageRefList) (err error) {
+func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 	fmt.Printf("Packages:\n")
 
 	if reflist == nil {
 		return
 	}
 
-	packageCollection := debian.NewPackageCollection(context.database)
-
 	err = reflist.ForEach(func(key []byte) error {
-		p, err := packageCollection.ByKey(key)
-		if err != nil {
-			return err
+		p, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
+		if err2 != nil {
+			return err2
 		}
 		fmt.Printf("  %s\n", p)
 		return nil
@@ -47,9 +45,9 @@ repositories, manage local repositories, filter them, merge,
 upgrade individual packages, take snapshots and publish them
 back as Debian repositories.
 
-aptly goal is to establish repeatiblity and controlled changes
-in package environment. aptly allows to fix set of packages in
-repository, so that package installation and upgrade becomes
+aptly's goal is to establish repeatability and controlled changes
+in a package-centric environment. aptly allows to fix a set of packages
+in a repository, so that package installation and upgrade becomes
 deterministic. At the same time aptly allows to perform controlled,
 fine-grained changes in repository contents to transition your
 package environment to new version.`,

cmd/context.go

@@ -2,14 +2,16 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/console"
 	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/s3"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -18,13 +20,17 @@ import (
 	"time"
 )
 
-// Common context shared by all commands
-var context struct {
+// AptlyContext is a common context shared by all commands
+type AptlyContext struct {
+	flags        *flag.FlagSet
+	configLoaded bool
+
 	progress          aptly.Progress
 	downloader        aptly.Downloader
 	database          database.Storage
 	packagePool       aptly.PackagePool
-	publishedStorage  aptly.PublishedStorage
+	publishedStorages map[string]aptly.PublishedStorage
+	collectionFactory *deb.CollectionFactory
 	dependencyOptions int
 	architecturesList []string
 	// Debug features
@@ -33,92 +39,195 @@ var context struct {
 	fileMemStats   *os.File
 }
 
-// InitContext initializes context with default settings
-func InitContext(cmd *commander.Command) error {
-	var err error
+var context *AptlyContext
 
-	context.dependencyOptions = 0
-	if utils.Config.DepFollowSuggests || cmd.Flag.Lookup("dep-follow-suggests").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowSuggests
-	}
-	if utils.Config.DepFollowRecommends || cmd.Flag.Lookup("dep-follow-recommends").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowRecommends
-	}
-	if utils.Config.DepFollowAllVariants || cmd.Flag.Lookup("dep-follow-all-variants").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowAllVariants
-	}
-	if utils.Config.DepFollowSource || cmd.Flag.Lookup("dep-follow-source").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowSource
+// Check interface
+var _ aptly.PublishedStorageProvider = &AptlyContext{}
+
+// FatalError is type for panicking to abort execution with non-zero
+// exit code and print meaningful explanation
+type FatalError struct {
+	ReturnCode int
+	Message    string
+}
+
+// Fatal panics and aborts execution with exit code 1
+func Fatal(err error) {
+	returnCode := 1
+	if err == commander.ErrFlagError || err == commander.ErrCommandError {
+		returnCode = 2
 	}
+	panic(&FatalError{ReturnCode: returnCode, Message: err.Error()})
+}
 
-	context.architecturesList = utils.Config.Architectures
-	optionArchitectures := cmd.Flag.Lookup("architectures").Value.String()
-	if optionArchitectures != "" {
-		context.architecturesList = strings.Split(optionArchitectures, ",")
-	}
+// Config loads and returns current configuration
+func (context *AptlyContext) Config() *utils.ConfigStructure {
+	if !context.configLoaded {
+		var err error
 
-	context.progress = console.NewProgress()
-	context.progress.Start()
+		configLocation := context.flags.Lookup("config").Value.String()
+		if configLocation != "" {
+			err = utils.LoadConfig(configLocation, &utils.Config)
 
-	context.downloader = http.NewDownloader(utils.Config.DownloadConcurrency, context.progress)
-
-	context.database, err = database.OpenDB(filepath.Join(utils.Config.RootDir, "db"))
-	if err != nil {
-		return fmt.Errorf("can't open database: %s", err)
-	}
-
-	context.packagePool = files.NewPackagePool(utils.Config.RootDir)
-	context.publishedStorage = files.NewPublishedStorage(utils.Config.RootDir)
-
-	if aptly.EnableDebug {
-		cpuprofile := cmd.Flag.Lookup("cpuprofile").Value.String()
-		if cpuprofile != "" {
-			context.fileCPUProfile, err = os.Create(cpuprofile)
 			if err != nil {
-				return err
+				Fatal(err)
 			}
-			pprof.StartCPUProfile(context.fileCPUProfile)
-		}
-
-		memprofile := cmd.Flag.Lookup("memprofile").Value.String()
-		if memprofile != "" {
-			context.fileMemProfile, err = os.Create(memprofile)
-			if err != nil {
-				return err
-			}
-		}
-
-		memstats := cmd.Flag.Lookup("memstats").Value.String()
-		if memstats != "" {
-			interval := cmd.Flag.Lookup("meminterval").Value.Get().(time.Duration)
-
-			context.fileMemStats, err = os.Create(memstats)
-			if err != nil {
-				return err
+		} else {
+			configLocations := []string{
+				filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
+				"/etc/aptly.conf",
 			}
 
-			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
-
-			go func() {
-				var stats runtime.MemStats
-
-				start := time.Now().UnixNano()
-
-				for {
-					runtime.ReadMemStats(&stats)
-					if context.fileMemStats != nil {
-						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
-							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
-						time.Sleep(interval)
-					} else {
-						break
-					}
+			for _, configLocation := range configLocations {
+				err = utils.LoadConfig(configLocation, &utils.Config)
+				if err == nil {
+					break
 				}
-			}()
+				if !os.IsNotExist(err) {
+					Fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
+				}
+			}
+
+			if err != nil {
+				fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
+				utils.SaveConfig(configLocations[0], &utils.Config)
+			}
+		}
+
+		context.configLoaded = true
+
+	}
+	return &utils.Config
+}
+
+// DependencyOptions calculates options related to dependecy handling
+func (context *AptlyContext) DependencyOptions() int {
+	if context.dependencyOptions == -1 {
+		context.dependencyOptions = 0
+		if context.Config().DepFollowSuggests || context.flags.Lookup("dep-follow-suggests").Value.Get().(bool) {
+			context.dependencyOptions |= deb.DepFollowSuggests
+		}
+		if context.Config().DepFollowRecommends || context.flags.Lookup("dep-follow-recommends").Value.Get().(bool) {
+			context.dependencyOptions |= deb.DepFollowRecommends
+		}
+		if context.Config().DepFollowAllVariants || context.flags.Lookup("dep-follow-all-variants").Value.Get().(bool) {
+			context.dependencyOptions |= deb.DepFollowAllVariants
+		}
+		if context.Config().DepFollowSource || context.flags.Lookup("dep-follow-source").Value.Get().(bool) {
+			context.dependencyOptions |= deb.DepFollowSource
 		}
 	}
 
-	return nil
+	return context.dependencyOptions
+}
+
+// ArchitecturesList returns list of architectures fixed via command line or config
+func (context *AptlyContext) ArchitecturesList() []string {
+	if context.architecturesList == nil {
+		context.architecturesList = context.Config().Architectures
+		optionArchitectures := context.flags.Lookup("architectures").Value.String()
+		if optionArchitectures != "" {
+			context.architecturesList = strings.Split(optionArchitectures, ",")
+		}
+	}
+
+	return context.architecturesList
+}
+
+// Progress creates or returns Progress object
+func (context *AptlyContext) Progress() aptly.Progress {
+	if context.progress == nil {
+		context.progress = console.NewProgress()
+		context.progress.Start()
+	}
+
+	return context.progress
+}
+
+// Downloader returns instance of current downloader
+func (context *AptlyContext) Downloader() aptly.Downloader {
+	if context.downloader == nil {
+		var downloadLimit int64
+		limitFlag := context.flags.Lookup("download-limit")
+		if limitFlag != nil {
+			downloadLimit = limitFlag.Value.Get().(int64)
+		}
+		if downloadLimit == 0 {
+			downloadLimit = context.Config().DownloadLimit
+		}
+		context.downloader = http.NewDownloader(context.Config().DownloadConcurrency,
+			downloadLimit*1024, context.Progress())
+	}
+
+	return context.downloader
+}
+
+// DBPath builds path to database
+func (context *AptlyContext) DBPath() string {
+	return filepath.Join(context.Config().RootDir, "db")
+}
+
+// Database opens and returns current instance of database
+func (context *AptlyContext) Database() (database.Storage, error) {
+	if context.database == nil {
+		var err error
+
+		context.database, err = database.OpenDB(context.DBPath())
+		if err != nil {
+			return nil, fmt.Errorf("can't open database: %s", err)
+		}
+	}
+
+	return context.database, nil
+}
+
+// CollectionFactory builds factory producing all kinds of collections
+func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
+	if context.collectionFactory == nil {
+		db, err := context.Database()
+		if err != nil {
+			Fatal(err)
+		}
+		context.collectionFactory = deb.NewCollectionFactory(db)
+	}
+
+	return context.collectionFactory
+}
+
+// PackagePool returns instance of PackagePool
+func (context *AptlyContext) PackagePool() aptly.PackagePool {
+	if context.packagePool == nil {
+		context.packagePool = files.NewPackagePool(context.Config().RootDir)
+	}
+
+	return context.packagePool
+}
+
+// PublishedStorage returns instance of PublishedStorage
+func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
+	publishedStorage, ok := context.publishedStorages[name]
+	if !ok {
+		if name == "" {
+			publishedStorage = files.NewPublishedStorage(context.Config().RootDir)
+		} else if strings.HasPrefix(name, "s3:") {
+			params, ok := context.Config().S3PublishRoots[name[3:]]
+			if !ok {
+				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
+			}
+
+			var err error
+			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
+				params.Region, params.Bucket, params.ACL, params.Prefix)
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			Fatal(fmt.Errorf("unknown published storage format: %v", name))
+		}
+		context.publishedStorages[name] = publishedStorage
+	}
+
+	return publishedStorage
 }
 
 // ShutdownContext shuts context down
@@ -149,3 +258,64 @@ func ShutdownContext() {
 		context.progress.Shutdown()
 	}
 }
+
+// InitContext initializes context with default settings
+func InitContext(flags *flag.FlagSet) error {
+	var err error
+
+	context = &AptlyContext{
+		flags:             flags,
+		dependencyOptions: -1,
+		publishedStorages: map[string]aptly.PublishedStorage{},
+	}
+
+	if aptly.EnableDebug {
+		cpuprofile := flags.Lookup("cpuprofile").Value.String()
+		if cpuprofile != "" {
+			context.fileCPUProfile, err = os.Create(cpuprofile)
+			if err != nil {
+				return err
+			}
+			pprof.StartCPUProfile(context.fileCPUProfile)
+		}
+
+		memprofile := flags.Lookup("memprofile").Value.String()
+		if memprofile != "" {
+			context.fileMemProfile, err = os.Create(memprofile)
+			if err != nil {
+				return err
+			}
+		}
+
+		memstats := flags.Lookup("memstats").Value.String()
+		if memstats != "" {
+			interval := flags.Lookup("meminterval").Value.Get().(time.Duration)
+
+			context.fileMemStats, err = os.Create(memstats)
+			if err != nil {
+				return err
+			}
+
+			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
+
+			go func() {
+				var stats runtime.MemStats
+
+				start := time.Now().UnixNano()
+
+				for {
+					runtime.ReadMemStats(&stats)
+					if context.fileMemStats != nil {
+						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
+							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
+						time.Sleep(interval)
+					} else {
+						break
+					}
+				}
+			}()
+		}
+	}
+
+	return nil
+}

cmd/db.go

@@ -1,8 +1,7 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
 )
 
 func makeCmdDb() *commander.Command {
@@ -11,7 +10,7 @@ func makeCmdDb() *commander.Command {
 		Short:     "manage aptly's internal database and package pool",
 		Subcommands: []*commander.Command{
 			makeCmdDbCleanup(),
+			makeCmdDbRecover(),
 		},
-		Flag: *flag.NewFlagSet("aptly-db", flag.ExitOnError),
 	}
 }

cmd/db_cleanup.go

@@ -2,10 +2,9 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
 	"sort"
 )
 
@@ -15,16 +14,15 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	// collect information about references packages...
-	existingPackageRefs := debian.NewPackageRefList()
+	existingPackageRefs := deb.NewPackageRefList()
 
-	context.progress.Printf("Loading mirrors, local repos and snapshots...\n")
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-	err = repoCollection.ForEach(func(repo *debian.RemoteRepo) error {
-		err := repoCollection.LoadComplete(repo)
+	context.Progress().Printf("Loading mirrors, local repos and snapshots...\n")
+	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
@@ -37,9 +35,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-	err = localRepoCollection.ForEach(func(repo *debian.LocalRepo) error {
-		err := localRepoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
@@ -52,9 +49,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	err = snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
-		err := snapshotCollection.LoadComplete(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return err
 		}
@@ -66,44 +62,45 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	}
 
 	// ... and compare it to the list of all packages
-	context.progress.Printf("Loading list of all packages...\n")
-	packageCollection := debian.NewPackageCollection(context.database)
-	allPackageRefs := packageCollection.AllPackageRefs()
+	context.Progress().Printf("Loading list of all packages...\n")
+	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
 
 	toDelete := allPackageRefs.Substract(existingPackageRefs)
 
 	// delete packages that are no longer referenced
-	context.progress.Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
+	context.Progress().Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
 
-	context.database.StartBatch()
+	// database can't err as collection factory already constructed
+	db, _ := context.Database()
+	db.StartBatch()
 	err = toDelete.ForEach(func(ref []byte) error {
-		return packageCollection.DeleteByKey(ref)
+		return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
 	})
 	if err != nil {
 		return err
 	}
 
-	err = context.database.FinishBatch()
+	err = db.FinishBatch()
 	if err != nil {
 		return fmt.Errorf("unable to write to DB: %s", err)
 	}
 
 	// now, build a list of files that should be present in Repository (package pool)
-	context.progress.Printf("Building list of files referenced by packages...\n")
+	context.Progress().Printf("Building list of files referenced by packages...\n")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())
-	context.progress.InitBar(int64(existingPackageRefs.Len()), false)
+	context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
 
 	err = existingPackageRefs.ForEach(func(key []byte) error {
-		pkg, err := packageCollection.ByKey(key)
-		if err != nil {
-			return err
+		pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
+		if err2 != nil {
+			return err2
 		}
-		paths, err := pkg.FilepathList(context.packagePool)
-		if err != nil {
-			return err
+		paths, err2 := pkg.FilepathList(context.PackagePool())
+		if err2 != nil {
+			return err2
 		}
 		referencedFiles = append(referencedFiles, paths...)
-		context.progress.AddBar(1)
+		context.Progress().AddBar(1)
 
 		return nil
 	})
@@ -112,11 +109,11 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	}
 
 	sort.Strings(referencedFiles)
-	context.progress.ShutdownBar()
+	context.Progress().ShutdownBar()
 
 	// build a list of files in the package pool
-	context.progress.Printf("Building list of files in package pool...\n")
-	existingFiles, err := context.packagePool.FilepathList(context.progress)
+	context.Progress().Printf("Building list of files in package pool...\n")
+	existingFiles, err := context.PackagePool().FilepathList(context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to collect file paths: %s", err)
 	}
@@ -125,24 +122,29 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
 
 	// delete files that are no longer referenced
-	context.progress.Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
+	context.Progress().Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
 
 	if len(filesToDelete) > 0 {
-		context.progress.InitBar(int64(len(filesToDelete)), false)
-		totalSize := int64(0)
+		context.Progress().InitBar(int64(len(filesToDelete)), false)
+
+		var size, totalSize int64
 		for _, file := range filesToDelete {
-			size, err := context.packagePool.Remove(file)
+			size, err = context.PackagePool().Remove(file)
 			if err != nil {
 				return err
 			}
 
-			context.progress.AddBar(1)
+			context.Progress().AddBar(1)
 			totalSize += size
 		}
-		context.progress.ShutdownBar()
+		context.Progress().ShutdownBar()
 
-		context.progress.Printf("Disk space freed: %.2f GiB...\n", float64(totalSize)/1024.0/1024.0/1024.0)
+		context.Progress().Printf("Disk space freed: %s...\n", utils.HumanBytes(totalSize))
 	}
+
+	context.Progress().Printf("Compacting database...\n")
+	err = db.CompactDB()
+
 	return err
 }
 
@@ -159,7 +161,6 @@ Example:
 
   $ aptly db cleanup
 `,
-		Flag: *flag.NewFlagSet("aptly-db-cleanup", flag.ExitOnError),
 	}
 
 	return cmd

cmd/db_recover.go

@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"github.com/smira/aptly/database"
+	"github.com/smira/commander"
+)
+
+// aptly db recover
+func aptlyDbRecover(cmd *commander.Command, args []string) error {
+	var err error
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	context.Progress().Printf("Recovering database...\n")
+	err = database.RecoverDB(context.DBPath())
+
+	return err
+}
+
+func makeCmdDbRecover() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyDbRecover,
+		UsageLine: "recover",
+		Short:     "recover DB after crash",
+		Long: `
+Database recover does its' best to recover the database after a crash.
+It is recommended to backup the DB before running recover.
+
+Example:
+
+  $ aptly db recover
+`,
+	}
+
+	return cmd
+}

cmd/graph.go

@@ -4,9 +4,8 @@ import (
 	"bytes"
 	"code.google.com/p/gographviz"
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"io"
 	"io/ioutil"
 	"os"
@@ -14,14 +13,15 @@ import (
 	"strings"
 )
 
-func graphvizEscape(s string) string {
-	return fmt.Sprintf("\"%s\"", strings.Replace(s, "\"", "\\\"", 0))
-}
-
 func aptlyGraph(cmd *commander.Command, args []string) error {
 	var err error
 
-	graph := gographviz.NewGraph()
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	graph := gographviz.NewEscape()
 	graph.SetDir(true)
 	graph.SetName("aptly")
 
@@ -29,21 +29,19 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Loading mirrors...\n")
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-
-	err = repoCollection.ForEach(func(repo *debian.RemoteRepo) error {
-		err := repoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
 
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
+		graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "darkgoldenrod1",
-			"label": graphvizEscape(fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
+			"label": fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
 				repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "),
-				strings.Join(repo.Architectures, ", "), repo.NumPackages())),
+				strings.Join(repo.Architectures, ", "), repo.NumPackages()),
 		})
 		existingNodes[repo.UUID] = true
 		return nil
@@ -55,20 +53,18 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Loading local repos...\n")
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-
-	err = localRepoCollection.ForEach(func(repo *debian.LocalRepo) error {
-		err := localRepoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
 
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
+		graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "mediumseagreen",
-			"label": graphvizEscape(fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
-				repo.Name, repo.Comment, repo.NumPackages())),
+			"label": fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
+				repo.Name, repo.Comment, repo.NumPackages()),
 		})
 		existingNodes[repo.UUID] = true
 		return nil
@@ -80,15 +76,13 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Loading snapshots...\n")
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-
-	snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
+	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
 		existingNodes[snapshot.UUID] = true
 		return nil
 	})
 
-	err = snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
-		err := snapshotCollection.LoadComplete(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return err
 		}
@@ -98,18 +92,18 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 			description = "Snapshot from repo"
 		}
 
-		graph.AddNode("aptly", graphvizEscape(snapshot.UUID), map[string]string{
+		graph.AddNode("aptly", snapshot.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "cadetblue1",
-			"label":     graphvizEscape(fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages())),
+			"label":     fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages()),
 		})
 
 		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
 			for _, uuid := range snapshot.SourceIDs {
 				_, exists := existingNodes[uuid]
 				if exists {
-					graph.AddEdge(graphvizEscape(uuid), "", graphvizEscape(snapshot.UUID), "", true, nil)
+					graph.AddEdge(uuid, snapshot.UUID, true, nil)
 				}
 			}
 		}
@@ -122,19 +116,20 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Loading published repos...\n")
 
-	publishedCollection := debian.NewPublishedRepoCollection(context.database)
-
-	publishedCollection.ForEach(func(repo *debian.PublishedRepo) error {
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
+	context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "darkolivegreen1",
-			"label":     graphvizEscape(fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution, repo.Component, strings.Join(repo.Architectures, ", "))),
+			"label": fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution,
+				strings.Join(repo.Components(), " "), strings.Join(repo.Architectures, ", ")),
 		})
 
-		_, exists := existingNodes[repo.SnapshotUUID]
-		if exists {
-			graph.AddEdge(graphvizEscape(repo.SnapshotUUID), "", graphvizEscape(repo.UUID), "", true, nil)
+		for _, uuid := range repo.Sources {
+			_, exists := existingNodes[uuid]
+			if exists {
+				graph.AddEdge(uuid, repo.UUID, true, nil)
+			}
 		}
 
 		return nil
@@ -198,13 +193,12 @@ func makeCmdGraph() *commander.Command {
 		Long: `
 Command graph displays relationship between mirrors, local repositories,
 snapshots and published repositories using graphviz package to render
-graph as image.
+graph as an image.
 
 Example:
 
   $ aptly graph
 `,
-		Flag: *flag.NewFlagSet("aptly-graph", flag.ExitOnError),
 	}
 
 	return cmd

cmd/mirror.go

@@ -1,19 +1,21 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"strings"
 )
 
-func getVerifier(cmd *commander.Command) (utils.Verifier, error) {
-	if utils.Config.GpgDisableVerify || cmd.Flag.Lookup("ignore-signatures").Value.Get().(bool) {
+func getVerifier(flags *flag.FlagSet) (utils.Verifier, error) {
+	if context.Config().GpgDisableVerify || flags.Lookup("ignore-signatures").Value.Get().(bool) {
 		return nil, nil
 	}
 
+	keyRings := flags.Lookup("keyring").Value.Get().([]string)
+
 	verifier := &utils.GpgVerifier{}
-	for _, keyRing := range keyRings.keyRings {
+	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}
 
@@ -42,8 +44,6 @@ func (k *keyRingsFlag) String() string {
 	return strings.Join(k.keyRings, ",")
 }
 
-var keyRings = keyRingsFlag{}
-
 func makeCmdMirror() *commander.Command {
 	return &commander.Command{
 		UsageLine: "mirror",
@@ -54,7 +54,8 @@ func makeCmdMirror() *commander.Command {
 			makeCmdMirrorShow(),
 			makeCmdMirrorDrop(),
 			makeCmdMirrorUpdate(),
+			makeCmdMirrorRename(),
+			makeCmdMirrorEdit(),
 		},
-		Flag: *flag.NewFlagSet("aptly-mirror", flag.ExitOnError),
 	}
 }

cmd/mirror_create.go

@@ -2,10 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"strings"
 )
 
@@ -13,10 +13,10 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	var err error
 	if !(len(args) == 2 && strings.HasPrefix(args[1], "ppa:") || len(args) >= 3) {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	downloadSources := utils.Config.DownloadSourcePackages || cmd.Flag.Lookup("with-sources").Value.Get().(bool)
+	downloadSources := context.Config().DownloadSourcePackages || context.flags.Lookup("with-sources").Value.Get().(bool)
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -25,7 +25,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 
 	mirrorName = args[0]
 	if len(args) == 2 {
-		archiveURL, distribution, components, err = debian.ParsePPA(args[1])
+		archiveURL, distribution, components, err = deb.ParsePPA(args[1], context.Config())
 		if err != nil {
 			return err
 		}
@@ -33,24 +33,32 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		archiveURL, distribution, components = args[1], args[2], args[3:]
 	}
 
-	repo, err := debian.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.architecturesList, downloadSources)
+	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(), downloadSources)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
-	verifier, err := getVerifier(cmd)
+	repo.Filter = context.flags.Lookup("filter").Value.String()
+	repo.FilterWithDeps = context.flags.Lookup("filter-with-deps").Value.Get().(bool)
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to create mirror: %s", err)
+		}
+	}
+
+	verifier, err := getVerifier(context.flags)
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.downloader, verifier)
+	err = repo.Fetch(context.Downloader(), verifier)
 	if err != nil {
 		return fmt.Errorf("unable to fetch mirror: %s", err)
 	}
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-
-	err = repoCollection.Add(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add mirror: %s", err)
 	}
@@ -66,7 +74,8 @@ func makeCmdMirrorCreate() *commander.Command {
 		Short:     "create new mirror",
 		Long: `
 Creates mirror <name> of remote repository, aptly supports both regular and flat Debian repositories exported
-via HTTP. aptly would try download Release file from remote repository and verify its signature.
+via HTTP. aptly would try download Release file from remote repository and verify its' signature. Command
+line format resembles apt utlitily sources.list(5).
 
 PPA urls could specified in short format:
 
@@ -81,7 +90,9 @@ Example:
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
-	cmd.Flag.Var(&keyRings, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd
 }

cmd/mirror_drop.go

@@ -2,30 +2,27 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-	repo, err := repoCollection.ByName(name)
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	force := cmd.Flag.Lookup("force").Value.Get().(bool)
+	force := context.flags.Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshotCollection := debian.NewSnapshotCollection(context.database)
-		snapshots := snapshotCollection.ByRemoteRepoSource(repo)
+		snapshots := context.CollectionFactory().SnapshotCollection().ByRemoteRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Mirror `%s` was used to create following snapshots:\n", repo.Name)
@@ -37,7 +34,7 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = repoCollection.Drop(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
@@ -54,7 +51,7 @@ func makeCmdMirrorDrop() *commander.Command {
 		Short:     "delete mirror",
 		Long: `
 Drop deletes information about remote repository mirror <name>. Package data is not deleted
-(it could be still used by other mirrors or snapshots).  If mirror is used as source
+(since it could still be used by other mirrors or snapshots).  If mirror is used as source
 to create a snapshot, aptly would refuse to delete such mirror, use flag -force to override.
 
 Example:

cmd/mirror_edit.go

@@ -0,0 +1,67 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	context.flags.Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "filter":
+			repo.Filter = flag.Value.String()
+		case "filter-with-deps":
+			repo.FilterWithDeps = flag.Value.Get().(bool)
+		}
+	})
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fmt.Printf("Mirror %s successfully updated.\n", repo)
+	return err
+}
+
+func makeCmdMirrorEdit() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorEdit,
+		UsageLine: "edit <name>",
+		Short:     "edit properties of mirorr",
+		Long: `
+Command edit allows to change settings of mirror:
+filters.
+
+Example:
+
+  $ aptly mirror edit -filter=nginx -filter-with-deps some-mirror
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-edit", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+
+	return cmd
+}

cmd/mirror_list.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"sort"
 )
 
@@ -12,29 +11,40 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	if repoCollection.Len() > 0 {
-		fmt.Printf("List of mirrors:\n")
-		repos := make([]string, repoCollection.Len())
-		i := 0
-		repoCollection.ForEach(func(repo *debian.RemoteRepo) error {
+	repos := make([]string, context.CollectionFactory().RemoteRepoCollection().Len())
+	i := 0
+	context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		if raw {
+			repos[i] = repo.Name
+		} else {
 			repos[i] = repo.String()
-			i++
-			return nil
-		})
-
-		sort.Strings(repos)
-		for _, repo := range repos {
-			fmt.Printf(" * %s\n", repo)
 		}
+		i++
+		return nil
+	})
 
-		fmt.Printf("\nTo get more information about mirror, run `aptly mirror show <name>`.\n")
+	sort.Strings(repos)
+
+	if raw {
+		for _, repo := range repos {
+			fmt.Printf("%s\n", repo)
+		}
 	} else {
-		fmt.Printf("No mirrors found, create one with `aptly mirror create ...`.\n")
+		if len(repos) > 0 {
+			fmt.Printf("List of mirrors:\n")
+			for _, repo := range repos {
+				fmt.Printf(" * %s\n", repo)
+			}
+
+			fmt.Printf("\nTo get more information about mirror, run `aptly mirror show <name>`.\n")
+		} else {
+			fmt.Printf("No mirrors found, create one with `aptly mirror create ...`.\n")
+		}
 	}
 	return err
 }
@@ -51,8 +61,9 @@ Example:
 
   $ aptly mirror list
 `,
-		Flag: *flag.NewFlagSet("aptly-mirror-list", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+
 	return cmd
 }

cmd/mirror_rename.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyMirrorRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.RemoteRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().RemoteRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: mirror %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nMirror %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdMirrorRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames mirror",
+		Long: `
+Command changes name of the mirror.Mirror name should be unique.
+
+Example:
+
+  $ aptly mirror rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/mirror_show.go

@@ -2,10 +2,9 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"strings"
 )
 
@@ -13,18 +12,17 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-	repo, err := repoCollection.ByName(name)
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = repoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -39,6 +37,14 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		downloadSources = "yes"
 	}
 	fmt.Printf("Download Sources: %s\n", downloadSources)
+	if repo.Filter != "" {
+		fmt.Printf("Filter: %s\n", repo.Filter)
+		filterWithDeps := "no"
+		if repo.FilterWithDeps {
+			filterWithDeps = "yes"
+		}
+		fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
+	}
 	if repo.LastDownloadDate.IsZero() {
 		fmt.Printf("Last update: never\n")
 	} else {
@@ -51,7 +57,7 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		fmt.Printf("%s: %s\n", k, repo.Meta[k])
 	}
 
-	withPackages := cmd.Flag.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		if repo.LastDownloadDate.IsZero() {
 			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
@@ -69,7 +75,7 @@ func makeCmdMirrorShow() *commander.Command {
 		UsageLine: "show <name>",
 		Short:     "show details about mirror",
 		Long: `
-Shows detailed information about mirror.
+Shows detailed information about the mirror.
 
 Example:
 

cmd/mirror_update.go

@@ -2,56 +2,64 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-	repo, err := repoCollection.ByName(name)
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = repoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	ignoreMismatch := cmd.Flag.Lookup("ignore-checksums").Value.Get().(bool)
+	ignoreMismatch := context.flags.Lookup("ignore-checksums").Value.Get().(bool)
 
-	verifier, err := getVerifier(cmd)
+	verifier, err := getVerifier(context.flags)
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.downloader, verifier)
+	err = repo.Fetch(context.Downloader(), verifier)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	packageCollection := debian.NewPackageCollection(context.database)
+	var filterQuery deb.PackageQuery
 
-	err = repo.Download(context.progress, context.downloader, packageCollection, context.packagePool, ignoreMismatch)
+	if repo.Filter != "" {
+		filterQuery, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
+
+	err = repo.Download(context.Progress(), context.Downloader(), context.CollectionFactory(), context.PackagePool(), ignoreMismatch,
+		context.DependencyOptions(), filterQuery)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = repoCollection.Update(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	context.progress.Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
+	context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
 	return err
 }
 
@@ -62,8 +70,8 @@ func makeCmdMirrorUpdate() *commander.Command {
 		Short:     "update mirror",
 		Long: `
 Updates remote mirror (downloads package files and meta information). When mirror is created,
-this command should be run for the first time to fetch mirror contents. This command could be
-run many times to get updated repository contents. If interrupted, command could be restarted safely.
+this command should be run for the first time to fetch mirror contents. This command can be
+run multiple times to get updated repository contents. If interrupted, command can be safely restarted.
 
 Example:
 
@@ -74,7 +82,8 @@ Example:
 
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
-	cmd.Flag.Var(&keyRings, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd
 }

cmd/publish.go

@@ -1,19 +1,20 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"strings"
 )
 
-func getSigner(cmd *commander.Command) (utils.Signer, error) {
-	if cmd.Flag.Lookup("skip-signing").Value.Get().(bool) || utils.Config.GpgDisableSign {
+func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
+	if flags.Lookup("skip-signing").Value.Get().(bool) || context.Config().GpgDisableSign {
 		return nil, nil
 	}
 
 	signer := &utils.GpgSigner{}
-	signer.SetKey(cmd.Flag.Lookup("gpg-key").Value.String())
-	signer.SetKeyRing(cmd.Flag.Lookup("keyring").Value.String(), cmd.Flag.Lookup("secret-keyring").Value.String())
+	signer.SetKey(flags.Lookup("gpg-key").Value.String())
+	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
 
 	err := signer.Init()
 	if err != nil {
@@ -24,15 +25,31 @@ func getSigner(cmd *commander.Command) (utils.Signer, error) {
 
 }
 
+func parsePrefix(param string) (storage, prefix string) {
+	i := strings.LastIndex(param, ":")
+	if i != -1 {
+		storage = param[:i]
+		prefix = param[i+1:]
+		if prefix == "" {
+			prefix = "."
+		}
+	} else {
+		prefix = param
+	}
+	return
+}
+
 func makeCmdPublish() *commander.Command {
 	return &commander.Command{
 		UsageLine: "publish",
 		Short:     "manage published repositories",
 		Subcommands: []*commander.Command{
-			makeCmdPublishSnapshot(),
-			makeCmdPublishList(),
 			makeCmdPublishDrop(),
+			makeCmdPublishList(),
+			makeCmdPublishRepo(),
+			makeCmdPublishSnapshot(),
+			makeCmdPublishSwitch(),
+			makeCmdPublishUpdate(),
 		},
-		Flag: *flag.NewFlagSet("aptly-publish", flag.ExitOnError),
 	}
 }

cmd/publish_drop.go

@@ -2,33 +2,32 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
 )
 
 func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 1 || len(args) > 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	if len(args) == 2 {
-		prefix = args[1]
+		param = args[1]
 	}
 
-	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
+	storage, prefix := parsePrefix(param)
 
-	err = publishedCollecton.Remove(context.publishedStorage, prefix, distribution)
+	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
+		context.CollectionFactory(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	fmt.Printf("\nPublished repositroy has been removed successfully.\n")
+	context.Progress().Printf("\nPublished repository has been removed successfully.\n")
 
 	return err
 }
@@ -36,17 +35,16 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 func makeCmdPublishDrop() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishDrop,
-		UsageLine: "drop <distribution> [<prefix>]",
+		UsageLine: "drop <distribution> [[<endpoint>:]<prefix>]",
 		Short:     "remove published repository",
 		Long: `
-Command removes whatever has been published under specified <prefix> and
-<distribution> name.
+Command removes whatever has been published under specified <prefix>,
+publishing <endpoint> and <distribution> name.
 
 Example:
 
     $ aptly publish drop wheezy
 `,
-		Flag: *flag.NewFlagSet("aptly-publish-drop", flag.ExitOnError),
 	}
 
 	return cmd

cmd/publish_list.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"sort"
 )
 
@@ -12,26 +11,24 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	if publishedCollecton.Len() == 0 {
-		fmt.Printf("No snapshots have been published. Publish a snapshot by running `aptly publish snapshot ...`.\n")
-		return err
-	}
+	published := make([]string, 0, context.CollectionFactory().PublishedRepoCollection().Len())
 
-	published := make([]string, 0, publishedCollecton.Len())
-
-	err = publishedCollecton.ForEach(func(repo *debian.PublishedRepo) error {
-		err := publishedCollecton.LoadComplete(repo, snapshotCollection)
+	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
 		if err != nil {
 			return err
 		}
 
-		published = append(published, repo.String())
+		if raw {
+			published = append(published, fmt.Sprintf("%s %s", repo.Prefix, repo.Distribution))
+		} else {
+			published = append(published, repo.String())
+		}
 		return nil
 	})
 
@@ -41,10 +38,21 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 
 	sort.Strings(published)
 
-	fmt.Printf("Published repositories:\n")
+	if raw {
+		for _, info := range published {
+			fmt.Printf("%s\n", info)
+		}
+	} else {
+		if len(published) == 0 {
+			fmt.Printf("No snapshots/local repos have been published. Publish a snapshot by running `aptly publish snapshot ...`.\n")
+			return err
+		}
 
-	for _, description := range published {
-		fmt.Printf("  * %s\n", description)
+		fmt.Printf("Published repositories:\n")
+
+		for _, description := range published {
+			fmt.Printf("  * %s\n", description)
+		}
 	}
 
 	return err
@@ -62,8 +70,9 @@ Example:
 
     $ aptly publish list
 `,
-		Flag: *flag.NewFlagSet("aptly-publish-list", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+
 	return cmd
 }

cmd/publish_repo.go

@@ -0,0 +1,45 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdPublishRepo() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSnapshotOrRepo,
+		UsageLine: "repo <name> [[<endpoint>:]<prefix>]",
+		Short:     "publish local repository",
+		Long: `
+Command publishes current state of local repository ready to be consumed
+by apt tools. Published repostiories appear under rootDir/public directory.
+Valid GPG key is required for publishing.
+
+Multiple component repository could be published by specifying several
+components split by commas via -component flag and multiple local
+repositories as the arguments:
+
+    aptly publish repo -component=main,contrib repo-main repo-contrib
+
+It is not recommended to publish local repositories directly unless the
+repository is for testing purposes and changes happen frequently. For
+production usage please take snapshot of repository and publish it
+using publish snapshot command.
+
+Example:
+
+    $ aptly publish repo testing
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-repo", flag.ExitOnError),
+	}
+	cmd.Flag.String("distribution", "", "distribution name to publish")
+	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("label", "", "label to publish")
+
+	return cmd
+}

cmd/publish_snapshot.go

@@ -2,124 +2,186 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"strings"
 )
 
-func aptlyPublishSnapshot(cmd *commander.Command, args []string) error {
+func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	var err error
-	if len(args) < 1 || len(args) > 2 {
+
+	components := strings.Split(context.flags.Lookup("component").Value.String(), ",")
+
+	if len(args) < len(components) || len(args) > len(components)+1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	name := args[0]
-
-	var prefix string
-	if len(args) == 2 {
-		prefix = args[1]
+	var param string
+	if len(args) == len(components)+1 {
+		param = args[len(components)]
+		args = args[0 : len(args)-1]
 	} else {
-		prefix = ""
+		param = ""
 	}
+	storage, prefix := parsePrefix(param)
 
-	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
+	var (
+		sources = []interface{}{}
+		message string
+	)
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	snapshot, err := snapshotCollection.ByName(name)
-	if err != nil {
-		return fmt.Errorf("unable to publish: %s", err)
-	}
+	if cmd.Name() == "snapshot" {
+		var (
+			snapshot     *deb.Snapshot
+			emptyWarning = false
+			parts        = []string{}
+		)
 
-	err = snapshotCollection.LoadComplete(snapshot)
-	if err != nil {
-		return fmt.Errorf("unable to publish: %s", err)
-	}
+		for _, name := range args {
+			snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
 
-	var sourceRepo *debian.RemoteRepo
+			err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
 
-	if snapshot.SourceKind == "repo" && len(snapshot.SourceIDs) == 1 {
-		repoCollection := debian.NewRemoteRepoCollection(context.database)
+			sources = append(sources, snapshot)
+			parts = append(parts, snapshot.Name)
 
-		sourceRepo, _ = repoCollection.ByUUID(snapshot.SourceIDs[0])
-	}
+			if snapshot.NumPackages() == 0 {
+				emptyWarning = true
+			}
+		}
 
-	component := cmd.Flag.Lookup("component").Value.String()
-	if component == "" {
-		if sourceRepo != nil && len(sourceRepo.Components) == 1 {
-			component = sourceRepo.Components[0]
+		if len(parts) == 1 {
+			message = fmt.Sprintf("Snapshot %s has", parts[0])
 		} else {
-			component = "main"
+			message = fmt.Sprintf("Snapshots %s have", strings.Join(parts, ", "))
+
 		}
+
+		if emptyWarning {
+			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
+		}
+	} else if cmd.Name() == "repo" {
+		var (
+			localRepo    *deb.LocalRepo
+			emptyWarning = false
+			parts        = []string{}
+		)
+
+		for _, name := range args {
+			localRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			err = context.CollectionFactory().LocalRepoCollection().LoadComplete(localRepo)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			sources = append(sources, localRepo)
+			parts = append(parts, localRepo.Name)
+
+			if localRepo.NumPackages() == 0 {
+				emptyWarning = true
+			}
+		}
+
+		if len(parts) == 1 {
+			message = fmt.Sprintf("Local repo %s has", parts[0])
+		} else {
+			message = fmt.Sprintf("Local repos %s have", strings.Join(parts, ", "))
+
+		}
+
+		if emptyWarning {
+			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
+		}
+	} else {
+		panic("unknown command")
 	}
 
-	distribution := cmd.Flag.Lookup("distribution").Value.String()
-	if distribution == "" {
-		if sourceRepo != nil {
-			distribution = sourceRepo.Distribution
-		}
+	distribution := context.flags.Lookup("distribution").Value.String()
 
-		if distribution == "" {
-			return fmt.Errorf("unable to guess distribution name, please specify explicitly")
-		}
-	}
-
-	published, err := debian.NewPublishedRepo(prefix, distribution, component, context.architecturesList, snapshot)
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
+	published.Origin = cmd.Flag.Lookup("origin").Value.String()
+	published.Label = cmd.Flag.Lookup("label").Value.String()
 
-	duplicate := publishedCollecton.CheckDuplicate(published)
+	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
-		publishedCollecton.LoadComplete(duplicate, snapshotCollection)
+		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
 		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 	}
 
-	signer, err := getSigner(cmd)
+	signer, err := getSigner(context.flags)
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	packageCollection := debian.NewPackageCollection(context.database)
-	err = published.Publish(context.packagePool, context.publishedStorage, packageCollection, signer, context.progress)
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = publishedCollecton.Add(published)
+	err = context.CollectionFactory().PublishedRepoCollection().Add(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	if prefix != "" && !strings.HasSuffix(prefix, "/") {
+	var repoComponents string
+	prefix, repoComponents, distribution = published.Prefix, strings.Join(published.Components(), " "), published.Distribution
+	if prefix == "." {
+		prefix = ""
+	} else if !strings.HasSuffix(prefix, "/") {
 		prefix += "/"
 	}
 
-	context.progress.Printf("\nSnapshot %s has been successfully published.\nPlease setup your webserver to serve directory '%s' with autoindexing.\n",
-		snapshot.Name, context.publishedStorage.PublicPath())
-	context.progress.Printf("Now you can add following line to apt sources:\n")
-	context.progress.Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, component)
-	if utils.StrSliceHasItem(published.Architectures, "source") {
-		context.progress.Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, component)
+	context.Progress().Printf("\n%s been successfully published.\n", message)
+
+	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.LocalPublishedStorage); ok {
+		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
+			localStorage.PublicPath())
 	}
-	context.progress.Printf("Don't forget to add your GPG key to apt with apt-key.\n")
-	context.progress.Printf("\nYou can also use `aptly serve` to publish your repositories over HTTP quickly.\n")
+
+	context.Progress().Printf("Now you can add following line to apt sources:\n")
+	context.Progress().Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
+	if utils.StrSliceHasItem(published.Architectures, "source") {
+		context.Progress().Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
+	}
+	context.Progress().Printf("Don't forget to add your GPG key to apt with apt-key.\n")
+	context.Progress().Printf("\nYou can also use `aptly serve` to publish your repositories over HTTP quickly.\n")
 
 	return err
 }
 
 func makeCmdPublishSnapshot() *commander.Command {
 	cmd := &commander.Command{
-		Run:       aptlyPublishSnapshot,
-		UsageLine: "snapshot <name> [<prefix>]",
+		Run:       aptlyPublishSnapshotOrRepo,
+		UsageLine: "snapshot <name> [[<endpoint>:]<prefix>]",
 		Short:     "publish snapshot",
 		Long: `
-Command publish publishes snapshot as Debian repository ready to be consumed
+Command publishes snapshot as Debian repository ready to be consumed
 by apt tools. Published repostiories appear under rootDir/public directory.
 Valid GPG key is required for publishing.
 
+Multiple component repository could be published by specifying several
+components split by commas via -component flag and multiple snapshots
+as the arguments:
+
+    aptly publish snapshot -component=main,contrib snap-main snap-contrib
+
 Example:
 
     $ aptly publish snapshot wheezy-main
@@ -127,11 +189,13 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-publish-snapshot", flag.ExitOnError),
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
-	cmd.Flag.String("component", "", "component name to publish")
+	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
-	cmd.Flag.String("keyring", "", "GPG keyring to use (instead of default)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("label", "", "label to publish")
 
 	return cmd
 }

cmd/publish_switch.go

@@ -0,0 +1,132 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"strings"
+)
+
+func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
+	var err error
+
+	components := strings.Split(context.flags.Lookup("component").Value.String(), ",")
+
+	if len(args) < len(components)+1 || len(args) > len(components)+2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	param := "."
+
+	var (
+		names    []string
+		snapshot *deb.Snapshot
+	)
+
+	if len(args) == len(components)+2 {
+		param = args[1]
+		names = args[2:]
+	} else {
+		names = args[1:]
+	}
+
+	storage, prefix := parsePrefix(param)
+
+	var published *deb.PublishedRepo
+
+	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	if published.SourceKind != "snapshot" {
+		return fmt.Errorf("unable to update: not a snapshot publish")
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	publishedComponents := published.Components()
+	if len(components) == 1 && len(publishedComponents) == 1 && components[0] == "" {
+		components = publishedComponents
+	}
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names))
+	}
+
+	for i, component := range components {
+		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(names[i])
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
+
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
+
+		published.UpdateSnapshot(component, snapshot)
+	}
+
+	signer, err := getSigner(context.flags)
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG signer: %s", err)
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
+
+	return err
+}
+
+func makeCmdPublishSwitch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSwitch,
+		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
+		Short:     "update published repository by switching to new snapshot",
+		Long: `
+Command switches in-place published repository with new snapshot contents. All
+publishing parameters are preserved (architecture list, distribution,
+component).
+
+For multiple component repositories, flag -component should be given with
+list of components to update. Corresponding snapshots should be given in the
+same order, e.g.:
+
+	aptly publish update -component=main,contrib wheezy wh-main wh-contrib
+
+Example:
+
+    $ aptly publish update wheezy ppa wheezy-7.5
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
+	}
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_update.go

@@ -0,0 +1,98 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+	storage, prefix := parsePrefix(param)
+
+	var published *deb.PublishedRepo
+
+	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	if published.SourceKind != "local" {
+		return fmt.Errorf("unable to update: not a local repository publish")
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	components := published.Components()
+	for _, component := range components {
+		published.UpdateLocalRepo(component)
+	}
+
+	signer, err := getSigner(context.flags)
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG signer: %s", err)
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
+
+	return err
+}
+
+func makeCmdPublishUpdate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishUpdate,
+		UsageLine: "update <distribution> [[<endpoint>:]<prefix>]",
+		Short:     "update published local repository",
+		Long: `
+Command re-publishes (updates) published local repository. <distribution>
+and <prefix> should be occupied with local repository published
+using command aptly publish repo. Update happens in-place with
+minimum possible downtime for published repository.
+
+For multiple component published repositories, all local repositories
+are updated.
+
+Example:
+
+    $ aptly publish update wheezy ppa
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError),
+	}
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+
+	return cmd
+}

cmd/repo.go

@@ -1,8 +1,7 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
 )
 
 func makeCmdRepo() *commander.Command {
@@ -14,12 +13,13 @@ func makeCmdRepo() *commander.Command {
 			makeCmdRepoCopy(),
 			makeCmdRepoCreate(),
 			makeCmdRepoDrop(),
+			makeCmdRepoEdit(),
 			makeCmdRepoImport(),
 			makeCmdRepoList(),
 			makeCmdRepoMove(),
 			makeCmdRepoRemove(),
 			makeCmdRepoShow(),
+			makeCmdRepoRename(),
 		},
-		Flag: *flag.NewFlagSet("aptly-repo", flag.ExitOnError),
 	}
 }

cmd/repo_add.go

@@ -2,10 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"os"
 	"path/filepath"
 	"sort"
@@ -16,44 +16,44 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
 	verifier := &utils.GpgVerifier{}
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-	repo, err := localRepoCollection.ByName(name)
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
-	err = localRepoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
-	context.progress.Printf("Loading packages...\n")
+	context.Progress().Printf("Loading packages...\n")
 
-	packageCollection := debian.NewPackageCollection(context.database)
-	list, err := debian.NewPackageListFromRefList(repo.RefList(), packageCollection, context.progress)
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
 	packageFiles := []string{}
+	failedFiles := []string{}
 
 	for _, location := range args[1:] {
-		info, err := os.Stat(location)
-		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to process %s: %s@|", location, err)
+		info, err2 := os.Stat(location)
+		if err2 != nil {
+			context.Progress().ColoredPrintf("@y[!]@| @!Unable to process %s: %s@|", location, err2)
+			failedFiles = append(failedFiles, location)
 			continue
 		}
 		if info.IsDir() {
-			err = filepath.Walk(location, func(path string, info os.FileInfo, err error) error {
-				if err != nil {
-					return err
+			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
+				if err3 != nil {
+					return err3
 				}
 				if info.IsDir() {
 					return nil
@@ -69,7 +69,8 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
 				packageFiles = append(packageFiles, location)
 			} else {
-				context.progress.ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
+				context.Progress().ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
+				failedFiles = append(failedFiles, location)
 				continue
 			}
 		}
@@ -80,46 +81,48 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	for _, file := range packageFiles {
 		var (
-			stanza debian.Stanza
-			err    error
-			p      *debian.Package
+			stanza deb.Stanza
+			p      *deb.Package
 		)
 
 		candidateProcessedFiles := []string{}
 		isSourcePackage := strings.HasSuffix(file, ".dsc")
 
 		if isSourcePackage {
-			stanza, err = debian.GetControlFileFromDsc(file, verifier)
+			stanza, err = deb.GetControlFileFromDsc(file, verifier)
 
 			if err == nil {
 				stanza["Package"] = stanza["Source"]
 				delete(stanza, "Source")
 
-				p, err = debian.NewSourcePackageFromControlFile(stanza)
+				p, err = deb.NewSourcePackageFromControlFile(stanza)
 			}
 		} else {
-			stanza, err = debian.GetControlFileFromDeb(file)
-			p = debian.NewPackageFromControlFile(stanza)
+			stanza, err = deb.GetControlFileFromDeb(file)
+			p = deb.NewPackageFromControlFile(stanza)
 		}
 		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
+			context.Progress().ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
+			failedFiles = append(failedFiles, file)
 			continue
 		}
 
-		checksums, err := utils.ChecksumsForFile(file)
+		var checksums utils.ChecksumInfo
+		checksums, err = utils.ChecksumsForFile(file)
 		if err != nil {
 			return err
 		}
 
 		if isSourcePackage {
-			p.UpdateFiles(append(p.Files(), debian.PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
+			p.UpdateFiles(append(p.Files(), deb.PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
 		} else {
-			p.UpdateFiles([]debian.PackageFile{debian.PackageFile{Filename: filepath.Base(file), Checksums: checksums}})
+			p.UpdateFiles([]deb.PackageFile{deb.PackageFile{Filename: filepath.Base(file), Checksums: checksums}})
 		}
 
-		err = context.packagePool.Import(file, checksums.MD5)
+		err = context.PackagePool().Import(file, checksums.MD5)
 		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", file, err)
+			context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", file, err)
+			failedFiles = append(failedFiles, file)
 			continue
 		}
 
@@ -131,9 +134,10 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 				continue
 			}
 			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
-			err = context.packagePool.Import(sourceFile, f.Checksums.MD5)
+			err = context.PackagePool().Import(sourceFile, f.Checksums.MD5)
 			if err != nil {
-				context.progress.ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", sourceFile, err)
+				context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", sourceFile, err)
+				failedFiles = append(failedFiles, file)
 				break
 			}
 
@@ -144,30 +148,32 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 			continue
 		}
 
-		err = packageCollection.Update(p)
+		err = context.CollectionFactory().PackageCollection().Update(p)
 		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to save package %s: %s@|", p, err)
+			context.Progress().ColoredPrintf("@y[!]@| @!Unable to save package %s: %s@|", p, err)
+			failedFiles = append(failedFiles, file)
 			continue
 		}
 
 		err = list.Add(p)
 		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
+			context.Progress().ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
+			failedFiles = append(failedFiles, file)
 			continue
 		}
 
-		context.progress.ColoredPrintf("@g[+]@| %s added@|", p)
+		context.Progress().ColoredPrintf("@g[+]@| %s added@|", p)
 		processedFiles = append(processedFiles, candidateProcessedFiles...)
 	}
 
-	repo.UpdateRefList(debian.NewPackageRefListFromPackageList(list))
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-	err = localRepoCollection.Update(repo)
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to save: %s", err)
 	}
 
-	if cmd.Flag.Lookup("remove-files").Value.Get().(bool) {
+	if context.flags.Lookup("remove-files").Value.Get().(bool) {
 		processedFiles = utils.StrSliceDeduplicate(processedFiles)
 
 		for _, file := range processedFiles {
@@ -178,6 +184,15 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		}
 	}
 
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
 	return err
 }
 
@@ -189,9 +204,9 @@ func makeCmdRepoAdd() *commander.Command {
 		Long: `
 Command adds packages to local repository from .deb (binary packages) and .dsc (source packages) files.
 When importing from directory aptly would do recursive scan looking for all files matching *.deb or *.dsc
-patterns. Every file discovered would be analyzed to extract metadata, package would be created and added
-to database. Files would be imported to internal package pool. For source packages, all required files are
-added as well automatically. Extra files for source package should be in the same directory as *.dsc file.
+patterns. Every file discovered would be analyzed to extract metadata, package would then be created and added
+to the database. Files would be imported to internal package pool. For source packages, all required files are
+added automatically as well. Extra files for source package should be in the same directory as *.dsc file.
 
 Example:
 

cmd/repo_copy.go

@@ -1,17 +1,17 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func makeCmdRepoCopy() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "copy <src-name> <dst-name> <package-spec> ...",
+		UsageLine: "copy <src-name> <dst-name> <package-query> ...",
 		Short:     "copy packages between local repositories",
 		Long: `
-Command copy copies packages matching <package-spec> from local repo
+Command copy copies packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
 Example:

cmd/repo_create.go

@@ -2,23 +2,23 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	repo := debian.NewLocalRepo(args[0], cmd.Flag.Lookup("comment").Value.String())
+	repo := deb.NewLocalRepo(args[0], context.flags.Lookup("comment").Value.String())
+	repo.DefaultDistribution = context.flags.Lookup("distribution").Value.String()
+	repo.DefaultComponent = context.flags.Lookup("component").Value.String()
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-
-	err = localRepoCollection.Add(repo)
+	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add local repo: %s", err)
 	}
@@ -45,6 +45,8 @@ Example:
 	}
 
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
+	cmd.Flag.String("distribution", "", "default distribution when publishing")
+	cmd.Flag.String("component", "main", "default component when publishing")
 
 	return cmd
 }

cmd/repo_drop.go

@@ -2,30 +2,41 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-	repo, err := localRepoCollection.ByName(name)
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	force := cmd.Flag.Lookup("force").Value.Get().(bool)
+	published := context.CollectionFactory().PublishedRepoCollection().ByLocalRepo(repo)
+	if len(published) > 0 {
+		fmt.Printf("Local repo `%s` is published currently:\n", repo.Name)
+		for _, repo := range published {
+			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+			if err != nil {
+				return fmt.Errorf("unable to load published: %s", err)
+			}
+			fmt.Printf(" * %s\n", repo)
+		}
+
+		return fmt.Errorf("unable to drop: local repo is published")
+	}
+
+	force := context.flags.Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshotCollection := debian.NewSnapshotCollection(context.database)
-		snapshots := snapshotCollection.ByLocalRepoSource(repo)
+		snapshots := context.CollectionFactory().SnapshotCollection().ByLocalRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Local repo `%s` was used to create following snapshots:\n", repo.Name)
@@ -37,7 +48,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = localRepoCollection.Drop(repo)
+	err = context.CollectionFactory().LocalRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
@@ -53,8 +64,8 @@ func makeCmdRepoDrop() *commander.Command {
 		UsageLine: "drop <name>",
 		Short:     "delete local repository",
 		Long: `
-Drop deletes information about local repo. Package data is not deleted
-(it could be still used by other mirrors or snapshots).
+Drop information about deletions from local repo. Package data is not deleted
+(since it could be still used by other mirrors or snapshots).
 
 Example:
 

cmd/repo_edit.go

@@ -0,0 +1,68 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoEdit(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	if context.flags.Lookup("comment").Value.String() != "" {
+		repo.Comment = context.flags.Lookup("comment").Value.String()
+	}
+
+	if context.flags.Lookup("distribution").Value.String() != "" {
+		repo.DefaultDistribution = context.flags.Lookup("distribution").Value.String()
+	}
+
+	if context.flags.Lookup("component").Value.String() != "" {
+		repo.DefaultComponent = context.flags.Lookup("component").Value.String()
+	}
+
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fmt.Printf("Local repo %s successfully updated.\n", repo)
+	return err
+}
+
+func makeCmdRepoEdit() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoEdit,
+		UsageLine: "edit <name>",
+		Short:     "edit properties of local repository",
+		Long: `
+Command edit allows to change metadata of local repository:
+comment, default distribution and component.
+
+Example:
+
+  $ aptly repo edit -distribution=wheezy testing
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-edit", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
+	cmd.Flag.String("distribution", "", "default distribution when publishing")
+	cmd.Flag.String("component", "", "default component when publishing")
+
+	return cmd
+}

cmd/repo_import.go

@@ -1,17 +1,17 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func makeCmdRepoImport() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "import <src-mirror> <dst-repo> <package-spec> ...",
+		UsageLine: "import <src-mirror> <dst-repo> <package-query> ...",
 		Short:     "import packages from mirror to local repository",
 		Long: `
-Command import looks up packages matching <package-spec> in mirror <src-mirror>
+Command import looks up packages matching <package-query> in mirror <src-mirror>
 and copies them to local repo <dst-repo>.
 
 Example:

cmd/repo_list.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"sort"
 )
 
@@ -12,35 +11,47 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	if localRepoCollection.Len() > 0 {
-		fmt.Printf("List of mirrors:\n")
-		repos := make([]string, localRepoCollection.Len())
-		i := 0
-		localRepoCollection.ForEach(func(repo *debian.LocalRepo) error {
-			err := localRepoCollection.LoadComplete(repo)
+	repos := make([]string, context.CollectionFactory().LocalRepoCollection().Len())
+	i := 0
+	context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		if raw {
+			repos[i] = repo.Name
+		} else {
+			err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 			if err != nil {
 				return err
 			}
 
 			repos[i] = fmt.Sprintf(" * %s (packages: %d)", repo.String(), repo.NumPackages())
-			i++
-			return nil
-		})
-
-		sort.Strings(repos)
-		for _, repo := range repos {
-			fmt.Println(repo)
 		}
+		i++
+		return nil
+	})
 
-		fmt.Printf("\nTo get more information about local repository, run `aptly repo show <name>`.\n")
+	sort.Strings(repos)
+
+	if raw {
+		for _, repo := range repos {
+			fmt.Printf("%s\n", repo)
+		}
 	} else {
-		fmt.Printf("No local repositories found, create one with `aptly repo create ...`.\n")
+		if len(repos) > 0 {
+			fmt.Printf("List of local repos:\n")
+			for _, repo := range repos {
+				fmt.Println(repo)
+			}
+
+			fmt.Printf("\nTo get more information about local repository, run `aptly repo show <name>`.\n")
+		} else {
+			fmt.Printf("No local repositories found, create one with `aptly repo create ...`.\n")
+		}
 	}
+
 	return err
 }
 
@@ -50,14 +61,15 @@ func makeCmdRepoList() *commander.Command {
 		UsageLine: "list",
 		Short:     "list local repositories",
 		Long: `
-List shows full list of local package repositories.
+List command shows full list of local package repositories.
 
 Example:
 
   $ aptly repo list
 `,
-		Flag: *flag.NewFlagSet("aptly-repo-list", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+
 	return cmd
 }

cmd/repo_move.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"sort"
 )
 
@@ -12,30 +13,28 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 3 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	command := cmd.Name()
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-
-	dstRepo, err := localRepoCollection.ByName(args[1])
+	dstRepo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	err = localRepoCollection.LoadComplete(dstRepo)
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(dstRepo)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
 	var (
-		srcRefList *debian.PackageRefList
-		srcRepo    *debian.LocalRepo
+		srcRefList *deb.PackageRefList
+		srcRepo    *deb.LocalRepo
 	)
 
 	if command == "copy" || command == "move" {
-		srcRepo, err = localRepoCollection.ByName(args[0])
+		srcRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -44,43 +43,42 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to %s: source and destination are the same", command)
 		}
 
-		err = localRepoCollection.LoadComplete(srcRepo)
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(srcRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
 		srcRefList = srcRepo.RefList()
 	} else if command == "import" {
-		repoCollection := debian.NewRemoteRepoCollection(context.database)
+		var srcRemoteRepo *deb.RemoteRepo
 
-		srcRepo, err := repoCollection.ByName(args[0])
+		srcRemoteRepo, err = context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		err = repoCollection.LoadComplete(srcRepo)
+		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(srcRemoteRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		if srcRepo.RefList() == nil {
+		if srcRemoteRepo.RefList() == nil {
 			return fmt.Errorf("unable to %s: mirror not updated", command)
 		}
 
-		srcRefList = srcRepo.RefList()
+		srcRefList = srcRemoteRepo.RefList()
 	} else {
 		panic("unexpected command")
 	}
 
-	context.progress.Printf("Loading packages...\n")
+	context.Progress().Printf("Loading packages...\n")
 
-	packageCollection := debian.NewPackageCollection(context.database)
-	dstList, err := debian.NewPackageListFromRefList(dstRepo.RefList(), packageCollection, context.progress)
+	dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	srcList, err := debian.NewPackageListFromRefList(srcRefList, packageCollection, context.progress)
+	srcList, err := deb.NewPackageListFromRefList(srcRefList, context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -89,14 +87,14 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	var architecturesList []string
 
-	withDeps := cmd.Flag.Lookup("with-deps").Value.Get().(bool)
+	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
 
 	if withDeps {
 		dstList.PrepareIndex()
 
 		// Calculate architectures
-		if len(context.architecturesList) > 0 {
-			architecturesList = context.architecturesList
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
 		} else {
 			architecturesList = dstList.Architectures(false)
 		}
@@ -108,7 +106,15 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	toProcess, err := srcList.Filter(args[2:], withDeps, dstList, context.dependencyOptions, architecturesList)
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i := 0; i < len(args)-2; i++ {
+		queries[i], err = query.Parse(args[i+2])
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+	}
+
+	toProcess, err := srcList.Filter(queries, withDeps, dstList, context.DependencyOptions(), architecturesList)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
@@ -123,7 +129,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		verb = "imported"
 	}
 
-	err = toProcess.ForEach(func(p *debian.Package) error {
+	err = toProcess.ForEach(func(p *deb.Package) error {
 		err = dstList.Add(p)
 		if err != nil {
 			return err
@@ -132,27 +138,27 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		if command == "move" {
 			srcList.Remove(p)
 		}
-		context.progress.ColoredPrintf("@g[o]@| %s %s", p, verb)
+		context.Progress().ColoredPrintf("@g[o]@| %s %s", p, verb)
 		return nil
 	})
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	if cmd.Flag.Lookup("dry-run").Value.Get().(bool) {
-		context.progress.Printf("\nChanges not saved, as dry run has been requested.\n")
+	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
-		dstRepo.UpdateRefList(debian.NewPackageRefListFromPackageList(dstList))
+		dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
 
-		err = localRepoCollection.Update(dstRepo)
+		err = context.CollectionFactory().LocalRepoCollection().Update(dstRepo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
 
 		if command == "move" {
-			srcRepo.UpdateRefList(debian.NewPackageRefListFromPackageList(srcList))
+			srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
 
-			err = localRepoCollection.Update(srcRepo)
+			err = context.CollectionFactory().LocalRepoCollection().Update(srcRepo)
 			if err != nil {
 				return fmt.Errorf("unable to save: %s", err)
 			}
@@ -165,10 +171,10 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 func makeCmdRepoMove() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "move <src-name> <dst-name> <package-spec> ...",
+		UsageLine: "move <src-name> <dst-name> <package-query> ...",
 		Short:     "move packages between local repositories",
 		Long: `
-Command move moves packages matching <package-spec> from local repo
+Command move moves packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
 Example:

cmd/repo_remove.go

@@ -2,57 +2,64 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-	repo, err := localRepoCollection.ByName(name)
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	err = localRepoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	context.progress.Printf("Loading packages...\n")
+	context.Progress().Printf("Loading packages...\n")
 
-	packageCollection := debian.NewPackageCollection(context.database)
-	list, err := debian.NewPackageListFromRefList(repo.RefList(), packageCollection, context.progress)
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
+	queries := make([]deb.PackageQuery, len(args)-1)
+	for i := 0; i < len(args)-1; i++ {
+		queries[i], err = query.Parse(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to remove: %s", err)
+		}
+	}
+
 	list.PrepareIndex()
-	toRemove, err := list.Filter(args[1:], false, nil, 0, nil)
+	toRemove, err := list.Filter(queries, false, nil, 0, nil)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	toRemove.ForEach(func(p *debian.Package) error {
+	toRemove.ForEach(func(p *deb.Package) error {
 		list.Remove(p)
-		context.progress.ColoredPrintf("@r[-]@| %s removed", p)
+		context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
 		return nil
 	})
 
-	if cmd.Flag.Lookup("dry-run").Value.Get().(bool) {
-		context.progress.Printf("\nChanges not saved, as dry run has been requested.\n")
+	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
-		repo.UpdateRefList(debian.NewPackageRefListFromPackageList(list))
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = localRepoCollection.Update(repo)
+		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
@@ -64,10 +71,10 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 func makeCmdRepoRemove() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoRemove,
-		UsageLine: "remove <name> <package-spec> ...",
+		UsageLine: "remove <name> <package-query> ...",
 		Short:     "remove packages from local repository",
 		Long: `
-Commands removes packages matching <package-spec> from local repository
+Commands removes packages matching <package-query> from local repository
 <name>. If removed packages are not referenced by other repos or
 snapshots, they can be removed completely (including files) by running
 'aptly db cleanup'.

cmd/repo_rename.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyRepoRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.LocalRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	repo, err = context.CollectionFactory().LocalRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().LocalRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: local repo %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nLocal repo %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdRepoRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames local repository",
+		Long: `
+Command changes name of the local repo. Local repo name should be unique.
+
+Example:
+
+  $ aptly repo rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/repo_show.go

@@ -2,36 +2,36 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-	repo, err := localRepoCollection.ByName(name)
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = localRepoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
 	fmt.Printf("Name: %s\n", repo.Name)
 	fmt.Printf("Comment: %s\n", repo.Comment)
+	fmt.Printf("Default Distribution: %s\n", repo.DefaultDistribution)
+	fmt.Printf("Default Component: %s\n", repo.DefaultComponent)
 	fmt.Printf("Number of packages: %d\n", repo.NumPackages())
 
-	withPackages := cmd.Flag.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		ListPackagesRefList(repo.RefList())
 	}
@@ -45,7 +45,7 @@ func makeCmdRepoShow() *commander.Command {
 		UsageLine: "show <name>",
 		Short:     "show details about local repository",
 		Long: `
-Show shows full information about local package repository.
+Show command shows full information about local package repository.
 
 ex:
   $ aptly repo show testing

cmd/serve.go

@@ -2,28 +2,32 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"net"
 	"net/http"
 	"os"
 	"sort"
+	"strings"
 )
 
 func aptlyServe(cmd *commander.Command, args []string) error {
 	var err error
 
-	publishedCollection := debian.NewPublishedRepoCollection(context.database)
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
 
-	if publishedCollection.Len() == 0 {
+	if context.CollectionFactory().PublishedRepoCollection().Len() == 0 {
 		fmt.Printf("No published repositories, unable to serve.\n")
 		return nil
 	}
 
-	listen := cmd.Flag.Lookup("listen").Value.String()
+	listen := context.flags.Lookup("listen").Value.String()
 
 	listenHost, listenPort, err := net.SplitHostPort(listen)
 
@@ -40,11 +44,11 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Serving published repositories, recommended apt sources list:\n\n")
 
-	sources := make(sort.StringSlice, 0, publishedCollection.Len())
-	published := make(map[string]*debian.PublishedRepo, publishedCollection.Len())
+	sources := make(sort.StringSlice, 0, context.CollectionFactory().PublishedRepoCollection().Len())
+	published := make(map[string]*deb.PublishedRepo, context.CollectionFactory().PublishedRepoCollection().Len())
 
-	err = publishedCollection.ForEach(func(repo *debian.PublishedRepo) error {
-		err := publishedCollection.LoadComplete(repo, snapshotCollection)
+	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
 		if err != nil {
 			return err
 		}
@@ -72,19 +76,20 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		}
 
 		fmt.Printf("# %s\ndeb http://%s:%s/%s %s %s\n",
-			repo, listenHost, listenPort, prefix, repo.Distribution, repo.Component)
+			repo, listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 
 		if utils.StrSliceHasItem(repo.Architectures, "source") {
 			fmt.Printf("deb-src http://%s:%s/%s %s %s\n",
-				listenHost, listenPort, prefix, repo.Distribution, repo.Component)
+				listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 		}
 	}
 
-	context.database.Close()
+	publicPath := context.GetPublishedStorage("").(aptly.LocalPublishedStorage).PublicPath()
+	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
 
-	err = http.ListenAndServe(listen, http.FileServer(http.Dir(context.publishedStorage.PublicPath())))
+	err = http.ListenAndServe(listen, http.FileServer(http.Dir(publicPath)))
 	if err != nil {
 		return fmt.Errorf("unable to serve: %s", err)
 	}

cmd/snapshot.go

@@ -1,8 +1,7 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
 )
 
 func makeCmdSnapshot() *commander.Command {
@@ -18,7 +17,7 @@ func makeCmdSnapshot() *commander.Command {
 			makeCmdSnapshotDiff(),
 			makeCmdSnapshotMerge(),
 			makeCmdSnapshotDrop(),
+			makeCmdSnapshotRename(),
 		},
-		Flag: *flag.NewFlagSet("aptly-snapshot", flag.ExitOnError),
 	}
 }

cmd/snapshot_create.go

@@ -2,52 +2,53 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 	var (
 		err      error
-		snapshot *debian.Snapshot
+		snapshot *deb.Snapshot
 	)
 
 	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" {
 		// aptly snapshot create snap from mirror mirror
+		var repo *deb.RemoteRepo
+
 		repoName, snapshotName := args[3], args[0]
 
-		repoCollection := debian.NewRemoteRepoCollection(context.database)
-		repo, err := repoCollection.ByName(repoName)
+		repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(repoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = repoCollection.LoadComplete(repo)
+		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		snapshot, err = debian.NewSnapshotFromRepository(snapshotName, repo)
+		snapshot, err = deb.NewSnapshotFromRepository(snapshotName, repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" {
 		// aptly snapshot create snap from repo repo
+		var repo *deb.LocalRepo
+
 		localRepoName, snapshotName := args[3], args[0]
 
-		localRepoCollection := debian.NewLocalRepoCollection(context.database)
-		repo, err := localRepoCollection.ByName(localRepoName)
+		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(localRepoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = localRepoCollection.LoadComplete(repo)
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		snapshot, err = debian.NewSnapshotFromLocalRepo(snapshotName, repo)
+		snapshot, err = deb.NewSnapshotFromLocalRepo(snapshotName, repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -55,17 +56,15 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		// aptly snapshot create snap empty
 		snapshotName := args[0]
 
-		packageList := debian.NewPackageList()
+		packageList := deb.NewPackageList()
 
-		snapshot = debian.NewSnapshotFromPackageList(snapshotName, nil, packageList, "Created as empty")
+		snapshot = deb.NewSnapshotFromPackageList(snapshotName, nil, packageList, "Created as empty")
 	} else {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-
-	err = snapshotCollection.Add(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().Add(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to add snapshot: %s", err)
 	}
@@ -97,7 +96,6 @@ Example:
 
   $ aptly snapshot create wheezy-main-today from mirror wheezy-main
 `,
-		Flag: *flag.NewFlagSet("aptly-snapshot-create", flag.ExitOnError),
 	}
 
 	return cmd

cmd/snapshot_diff.go

@@ -2,55 +2,51 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	onlyMatching := cmd.Flag.Lookup("only-matching").Value.Get().(bool)
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	packageCollection := debian.NewPackageCollection(context.database)
+	onlyMatching := context.flags.Lookup("only-matching").Value.Get().(bool)
 
 	// Load <name-a> snapshot
-	snapshotA, err := snapshotCollection.ByName(args[0])
+	snapshotA, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(snapshotA)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotA)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
 	// Load <name-b> snapshot
-	snapshotB, err := snapshotCollection.ByName(args[1])
+	snapshotB, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(snapshotB)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotB)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
 	// Calculate diff
-	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), packageCollection)
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
 	if err != nil {
 		return fmt.Errorf("unable to calculate diff: %s", err)
 	}
 
 	if len(diff) == 0 {
-		context.progress.Printf("Snapshots are identical.\n")
+		context.Progress().Printf("Snapshots are identical.\n")
 	} else {
-		context.progress.Printf("  Arch   | Package                                  | Version in A                             | Version in B\n")
+		context.Progress().Printf("  Arch   | Package                                  | Version in A                             | Version in B\n")
 		for _, pdiff := range diff {
 			if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) {
 				continue
@@ -84,7 +80,7 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 				}
 			}
 
-			context.progress.ColoredPrintf(code+" %-6s | %-40s | %-40s | %-40s", arch, pkg, verA, verB)
+			context.Progress().ColoredPrintf(code+" %-6s | %-40s | %-40s | %-40s", arch, pkg, verA, verB)
 		}
 	}
 

cmd/snapshot_drop.go

@@ -2,33 +2,30 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	snapshot, err := snapshotCollection.ByName(name)
+	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	publishedRepoCollection := debian.NewPublishedRepoCollection(context.database)
-	published := publishedRepoCollection.BySnapshot(snapshot)
+	published := context.CollectionFactory().PublishedRepoCollection().BySnapshot(snapshot)
 
 	if len(published) > 0 {
 		fmt.Printf("Snapshot `%s` is published currently:\n", snapshot.Name)
 		for _, repo := range published {
-			err = publishedRepoCollection.LoadComplete(repo, snapshotCollection)
+			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
 			if err != nil {
 				return fmt.Errorf("unable to load published: %s", err)
 			}
@@ -38,9 +35,9 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: snapshot is published")
 	}
 
-	force := cmd.Flag.Lookup("force").Value.Get().(bool)
+	force := context.flags.Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := snapshotCollection.BySnapshotSource(snapshot)
+		snapshots := context.CollectionFactory().SnapshotCollection().BySnapshotSource(snapshot)
 		if len(snapshots) > 0 {
 			fmt.Printf("Snapshot `%s` was used as a source in following snapshots:\n", snapshot.Name)
 			for _, snap := range snapshots {
@@ -51,7 +48,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = snapshotCollection.Drop(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().Drop(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
@@ -67,7 +64,7 @@ func makeCmdSnapshotDrop() *commander.Command {
 		UsageLine: "drop <name>",
 		Short:     "delete snapshot",
 		Long: `
-Drop removes information about snapshot. If snapshot is published,
+Drop removes information about a snapshot. If snapshot is published,
 it can't be dropped.
 
 Example:

cmd/snapshot_list.go

@@ -2,41 +2,94 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"sort"
 )
 
+// Snapshot sorting methods
+const (
+	SortName = iota
+	SortTime
+)
+
+type snapshotListToSort struct {
+	list       []*deb.Snapshot
+	sortMethod int
+}
+
+func parseSortMethod(sortMethod string) (int, error) {
+	switch sortMethod {
+	case "time", "Time":
+		return SortTime, nil
+	case "name", "Name":
+		return SortName, nil
+	}
+
+	return -1, fmt.Errorf("sorting method \"%s\" unknown", sortMethod)
+}
+
+func (s snapshotListToSort) Swap(i, j int) {
+	s.list[i], s.list[j] = s.list[j], s.list[i]
+}
+
+func (s snapshotListToSort) Less(i, j int) bool {
+	switch s.sortMethod {
+	case SortName:
+		return s.list[i].Name < s.list[j].Name
+	case SortTime:
+		return s.list[i].CreatedAt.Before(s.list[j].CreatedAt)
+	}
+	panic("unknown sort method")
+}
+
+func (s snapshotListToSort) Len() int {
+	return len(s.list)
+}
+
 func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
+
+	snapshotsToSort := &snapshotListToSort{}
+	snapshotsToSort.list = make([]*deb.Snapshot, context.CollectionFactory().SnapshotCollection().Len())
+	snapshotsToSort.sortMethod, err = parseSortMethod(sortMethodString)
+	if err != nil {
 		return err
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	i := 0
+	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		snapshotsToSort.list[i] = snapshot
+		i++
 
-	if snapshotCollection.Len() > 0 {
-		fmt.Printf("List of snapshots:\n")
+		return nil
+	})
 
-		snapshots := make([]string, snapshotCollection.Len())
+	sort.Sort(snapshotsToSort)
 
-		i := 0
-		snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
-			snapshots[i] = snapshot.String()
-			i++
-			return nil
-		})
-
-		sort.Strings(snapshots)
-		for _, snapshot := range snapshots {
-			fmt.Printf(" * %s\n", snapshot)
+	if raw {
+		for _, snapshot := range snapshotsToSort.list {
+			fmt.Printf("%s\n", snapshot.Name)
 		}
-
-		fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
 	} else {
-		fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n")
+		if len(snapshotsToSort.list) > 0 {
+			fmt.Printf("List of snapshots:\n")
+
+			for _, snapshot := range snapshotsToSort.list {
+				fmt.Printf(" * %s\n", snapshot.String())
+			}
+
+			fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
+		} else {
+			fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n")
+		}
 	}
 	return err
 
@@ -54,8 +107,10 @@ Example:
 
   $ aptly snapshot list
 `,
-		Flag: *flag.NewFlagSet("aptly-snapshot-list", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+	cmd.Flag.String("sort", "name", "display list in 'name' or creation 'time' order")
+
 	return cmd
 }

cmd/snapshot_merge.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"strings"
 )
 
@@ -12,29 +11,39 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-
-	sources := make([]*debian.Snapshot, len(args)-1)
+	sources := make([]*deb.Snapshot, len(args)-1)
 
 	for i := 0; i < len(args)-1; i++ {
-		sources[i], err = snapshotCollection.ByName(args[i+1])
+		sources[i], err = context.CollectionFactory().SnapshotCollection().ByName(args[i+1])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
 
-		err = snapshotCollection.LoadComplete(sources[i])
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(sources[i])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
 	}
 
-	result := sources[0].RefList()
+	latest := context.flags.Lookup("latest").Value.Get().(bool)
+    noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
 
+    if noRemove && latest {
+        return fmt.Errorf("-no-remove and -latest can't be specified together")
+    }
+
+	overrideMatching := !latest && !noRemove
+
+	result := sources[0].RefList()
 	for i := 1; i < len(sources); i++ {
-		result = result.Merge(sources[i].RefList(), true)
+		result = result.Merge(sources[i].RefList(), overrideMatching)
+	}
+
+	if latest {
+		deb.FilterLatestRefs(result)
 	}
 
 	sourceDescription := make([]string, len(sources))
@@ -43,10 +52,10 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	}
 
 	// Create <destination> snapshot
-	destination := debian.NewSnapshotFromRefList(args[0], sources, result,
+	destination := deb.NewSnapshotFromRefList(args[0], sources, result,
 		fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
-	err = snapshotCollection.Add(destination)
+	err = context.CollectionFactory().SnapshotCollection().Add(destination)
 	if err != nil {
 		return fmt.Errorf("unable to create snapshot: %s", err)
 	}
@@ -62,18 +71,20 @@ func makeCmdSnapshotMerge() *commander.Command {
 		UsageLine: "merge <destination> <source> [<source>...]",
 		Short:     "merges snapshots",
 		Long: `
-Merge merges several <source> snapshots into one <destination> snapshot.
-Merge happens from left to right. Packages with the same name-architecture
-pair are replaced during merge (package from latest snapshot on the list
-wins). If run with only one source snapshot, merge copies <source> into
+Merge command merges several <source> snapshots into one <destination> snapshot.
+Merge happens from left to right. By default, packages with the same
+name-architecture pair are replaced during merge (package from latest snapshot
+on the list wins).  If run with only one source snapshot, merge copies <source> into
 <destination>.
 
 Example:
 
     $ aptly snapshot merge wheezy-w-backports wheezy-main wheezy-backports
 `,
-		Flag: *flag.NewFlagSet("aptly-snapshot-merge", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("latest", false, "use only the latest version of each package")
+    cmd.Flag.Bool("no-remove", false, "don't remove duplicate arch/name packages")
+
 	return cmd
 }

cmd/snapshot_pull.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"sort"
 	"strings"
 )
@@ -13,61 +14,59 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 4 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	noDeps := cmd.Flag.Lookup("no-deps").Value.Get().(bool)
-	noRemove := cmd.Flag.Lookup("no-remove").Value.Get().(bool)
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	packageCollection := debian.NewPackageCollection(context.database)
+	noDeps := context.flags.Lookup("no-deps").Value.Get().(bool)
+	noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
+	allMatches := context.flags.Lookup("all-matches").Value.Get().(bool)
 
 	// Load <name> snapshot
-	snapshot, err := snapshotCollection.ByName(args[0])
+	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
 	// Load <source> snapshot
-	source, err := snapshotCollection.ByName(args[1])
+	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(source)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	context.progress.Printf("Dependencies would be pulled into snapshot:\n    %s\nfrom snapshot:\n    %s\nand result would be saved as new snapshot %s.\n",
+	context.Progress().Printf("Dependencies would be pulled into snapshot:\n    %s\nfrom snapshot:\n    %s\nand result would be saved as new snapshot %s.\n",
 		snapshot, source, args[2])
 
 	// Convert snapshot to package list
-	context.progress.Printf("Loading packages (%d)...\n", snapshot.RefList().Len()+source.RefList().Len())
-	packageList, err := debian.NewPackageListFromRefList(snapshot.RefList(), packageCollection, context.progress)
+	context.Progress().Printf("Loading packages (%d)...\n", snapshot.RefList().Len()+source.RefList().Len())
+	packageList, err := deb.NewPackageListFromRefList(snapshot.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	sourcePackageList, err := debian.NewPackageListFromRefList(source.RefList(), packageCollection, context.progress)
+	sourcePackageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	context.progress.Printf("Building indexes...\n")
+	context.Progress().Printf("Building indexes...\n")
 	packageList.PrepareIndex()
 	sourcePackageList.PrepareIndex()
 
 	// Calculate architectures
 	var architecturesList []string
 
-	if len(context.architecturesList) > 0 {
-		architecturesList = context.architecturesList
+	if len(context.ArchitecturesList()) > 0 {
+		architecturesList = context.ArchitecturesList()
 	} else {
 		architecturesList = packageList.Architectures(false)
 	}
@@ -78,90 +77,71 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
 	}
 
-	// Initial dependencies out of arguments
-	initialDependencies := make([]debian.Dependency, len(args)-3)
-	for i, arg := range args[3:] {
-		initialDependencies[i], err = debian.ParseDependency(arg)
-		if err != nil {
-			return fmt.Errorf("unable to parse argument: %s", err)
-		}
-	}
-
-	// Perform pull
+	// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
+	var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
 	for _, arch := range architecturesList {
-		dependencies := make([]debian.Dependency, len(initialDependencies), 128)
-		for i := range dependencies {
-			dependencies[i] = initialDependencies[i]
-			dependencies[i].Architecture = arch
-		}
-
-		// Go over list of initial dependencies + list of dependencies found
-		for i := 0; i < len(dependencies); i++ {
-			dep := dependencies[i]
-
-			// Search for package that can satisfy dependencies
-			pkg := sourcePackageList.Search(dep)
-			if pkg == nil {
-				context.progress.ColoredPrintf("@y[!]@| @!Dependency %s can't be satisfied with source %s@|", &dep, source)
-				continue
-			}
-
-			if !noRemove {
-				// Remove all packages with the same name and architecture
-				for p := packageList.Search(debian.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}); p != nil; {
-					packageList.Remove(p)
-					context.progress.ColoredPrintf("@r[-]@| %s removed", p)
-					p = packageList.Search(debian.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name})
-				}
-			}
-
-			// Add new discovered package
-			packageList.Add(pkg)
-			context.progress.ColoredPrintf("@g[+]@| %s added", pkg)
-
-			if noDeps {
-				continue
-			}
-
-			// Find missing dependencies for single added package
-			pL := debian.NewPackageList()
-			pL.Add(pkg)
-
-			missing, err := pL.VerifyDependencies(context.dependencyOptions, []string{arch}, packageList, nil)
-			if err != nil {
-				context.progress.ColoredPrintf("@y[!]@| @!Error while verifying dependencies for pkg %s: %s@|", pkg, err)
-			}
-
-			// Append missing dependencies to the list of dependencies to satisfy
-			for _, misDep := range missing {
-				found := false
-				for _, d := range dependencies {
-					if d == misDep {
-						found = true
-						break
-					}
-				}
-
-				if !found {
-					dependencies = append(dependencies, misDep)
-				}
-			}
-		}
+		archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
 	}
 
-	if cmd.Flag.Lookup("dry-run").Value.Get().(bool) {
-		context.progress.Printf("\nNot creating snapshot, as dry run was requested.\n")
+	// Initial queries out of arguments
+	queries := make([]deb.PackageQuery, len(args)-3)
+	for i, arg := range args[3:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+		// Add architecture filter
+		queries[i] = &deb.AndQuery{queries[i], archQuery}
+	}
+
+	// Filter with dependencies as requested
+	result, err := sourcePackageList.Filter(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+	result.PrepareIndex()
+
+	alreadySeen := map[string]bool{}
+
+	result.ForEachIndexed(func(pkg *deb.Package) error {
+		key := pkg.Architecture + "_" + pkg.Name
+		_, seen := alreadySeen[key]
+
+		// If we haven't seen such name-architecture pair and were instructed to remove, remove it
+		if !noRemove && !seen {
+			// Remove all packages with the same name and architecture
+			pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
+			for _, p := range pS {
+				packageList.Remove(p)
+				context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
+			}
+		}
+
+		// If !allMatches, add only first matching name-arch package
+		if !seen || allMatches {
+			packageList.Add(pkg)
+			context.Progress().ColoredPrintf("@g[+]@| %s added", pkg)
+		}
+
+		alreadySeen[key] = true
+
+		return nil
+	})
+	alreadySeen = nil
+
+	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+		context.Progress().Printf("\nNot creating snapshot, as dry run was requested.\n")
 	} else {
 		// Create <destination> snapshot
-		destination := debian.NewSnapshotFromPackageList(args[2], []*debian.Snapshot{snapshot, source}, packageList,
+		destination := deb.NewSnapshotFromPackageList(args[2], []*deb.Snapshot{snapshot, source}, packageList,
 			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", snapshot.Name, source.Name, strings.Join(args[3:], " ")))
 
-		err = snapshotCollection.Add(destination)
+		err = context.CollectionFactory().SnapshotCollection().Add(destination)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		context.progress.Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+		context.Progress().Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
 	}
 	return err
 }
@@ -169,14 +149,14 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 func makeCmdSnapshotPull() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotPull,
-		UsageLine: "pull <name> <source> <destination> <package-name> ...",
+		UsageLine: "pull <name> <source> <destination> <package-query> ...",
 		Short:     "pull packages from another snapshot",
 		Long: `
-Command pull pulls new packages along with its dependencies to snapshot <name>
+Command pull pulls new packages along with its' dependencies to snapshot <name>
 from snapshot <source>. Pull can upgrade package version in <name> with
 versions from <source> following dependencies. New snapshot <destination>
-is created as result of this process. Packages could be specified simply
-as 'package-name' or as dependency 'package-name (>= version)'.
+is created as a result of this process. Packages could be specified simply
+as 'package-name' or as package queries.
 
 Example:
 
@@ -188,6 +168,7 @@ Example:
 	cmd.Flag.Bool("dry-run", false, "don't create destination snapshot, just show what would be pulled")
 	cmd.Flag.Bool("no-deps", false, "don't process dependencies, just pull listed packages")
 	cmd.Flag.Bool("no-remove", false, "don't remove other package versions when pulling package")
+	cmd.Flag.Bool("all-matches", false, "pull all the packages that satisfy the dependency version requirements")
 
 	return cmd
 }

cmd/snapshot_rename.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlySnapshotRename(cmd *commander.Command, args []string) error {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().SnapshotCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: snapshot %s already exists", newName)
+	}
+
+	snapshot.Name = newName
+	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nSnapshot %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdSnapshotRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames snapshot",
+		Long: `
+Command changes name of the snapshot. Snapshot name should be unique.
+
+Example:
+
+  $ aptly snapshot rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/snapshot_show.go

@@ -2,27 +2,25 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	snapshot, err := snapshotCollection.ByName(name)
+	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -32,7 +30,7 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Description: %s\n", snapshot.Description)
 	fmt.Printf("Number of packages: %d\n", snapshot.NumPackages())
 
-	withPackages := cmd.Flag.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		ListPackagesRefList(snapshot.RefList())
 	}
@@ -46,7 +44,7 @@ func makeCmdSnapshotShow() *commander.Command {
 		UsageLine: "show <name>",
 		Short:     "shows details about snapshot",
 		Long: `
-Command show displays full information about snapshot.
+Command show displays full information about a snapshot.
 
 Example:
 

cmd/snapshot_verify.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"sort"
 )
 
@@ -12,40 +11,38 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	packageCollection := debian.NewPackageCollection(context.database)
-
-	snapshots := make([]*debian.Snapshot, len(args))
+	snapshots := make([]*deb.Snapshot, len(args))
 	for i := range snapshots {
-		snapshots[i], err = snapshotCollection.ByName(args[i])
+		snapshots[i], err = context.CollectionFactory().SnapshotCollection().ByName(args[i])
 		if err != nil {
 			return fmt.Errorf("unable to verify: %s", err)
 		}
 
-		err = snapshotCollection.LoadComplete(snapshots[i])
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshots[i])
 		if err != nil {
 			return fmt.Errorf("unable to verify: %s", err)
 		}
 	}
 
-	context.progress.Printf("Loading packages...\n")
+	context.Progress().Printf("Loading packages...\n")
 
-	packageList, err := debian.NewPackageListFromRefList(snapshots[0].RefList(), packageCollection, context.progress)
+	packageList, err := deb.NewPackageListFromRefList(snapshots[0].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	sourcePackageList := debian.NewPackageList()
+	sourcePackageList := deb.NewPackageList()
 	err = sourcePackageList.Append(packageList)
 	if err != nil {
 		fmt.Errorf("unable to merge sources: %s", err)
 	}
 
+	var pL *deb.PackageList
 	for i := 1; i < len(snapshots); i++ {
-		pL, err := debian.NewPackageListFromRefList(snapshots[i].RefList(), packageCollection, context.progress)
+		pL, err = deb.NewPackageListFromRefList(snapshots[i].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 		if err != nil {
 			fmt.Errorf("unable to load packages: %s", err)
 		}
@@ -60,8 +57,8 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 
 	var architecturesList []string
 
-	if len(context.architecturesList) > 0 {
-		architecturesList = context.architecturesList
+	if len(context.ArchitecturesList()) > 0 {
+		architecturesList = context.ArchitecturesList()
 	} else {
 		architecturesList = packageList.Architectures(true)
 	}
@@ -70,17 +67,17 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
 	}
 
-	context.progress.Printf("Verifying...\n")
+	context.Progress().Printf("Verifying...\n")
 
-	missing, err := packageList.VerifyDependencies(context.dependencyOptions, architecturesList, sourcePackageList, context.progress)
+	missing, err := packageList.VerifyDependencies(context.DependencyOptions(), architecturesList, sourcePackageList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to verify dependencies: %s", err)
 	}
 
 	if len(missing) == 0 {
-		context.progress.Printf("All dependencies are satisfied.\n")
+		context.Progress().Printf("All dependencies are satisfied.\n")
 	} else {
-		context.progress.Printf("Missing dependencies (%d):\n", len(missing))
+		context.Progress().Printf("Missing dependencies (%d):\n", len(missing))
 		deps := make([]string, len(missing))
 		i := 0
 		for _, dep := range missing {
@@ -91,7 +88,7 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 		sort.Strings(deps)
 
 		for _, dep := range deps {
-			context.progress.Printf("  %s\n", dep)
+			context.Progress().Printf("  %s\n", dep)
 		}
 	}
 
@@ -104,7 +101,7 @@ func makeCmdSnapshotVerify() *commander.Command {
 		UsageLine: "verify <name> [<source> ...]",
 		Short:     "verify dependencies in snapshot",
 		Long: `
-Verify does depenency resolution in snapshot <name>, possibly using additional
+Verify does dependency resolution in snapshot <name>, possibly using additional
 snapshots <source> as dependency sources. All unsatisfied dependencies are
 printed.
 
@@ -112,7 +109,6 @@ Example:
 
     $ aptly snapshot verify wheezy-main wheezy-contrib wheezy-non-free
 `,
-		Flag: *flag.NewFlagSet("aptly-snapshot-verify", flag.ExitOnError),
 	}
 
 	return cmd

cmd/version.go

@@ -2,12 +2,16 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/commander"
 )
 
 func aptlyVersion(cmd *commander.Command, args []string) error {
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
 	fmt.Printf("aptly version: %s\n", aptly.Version)
 	return nil
 }
@@ -23,6 +27,5 @@ Shows aptly version.
 ex:
   $ aptly version
 `,
-		Flag: *flag.NewFlagSet("aptly-version", flag.ExitOnError),
 	}
 }

database/leveldb.go

@@ -7,6 +7,8 @@ import (
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/filter"
 	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 // Errors for Storage
@@ -24,6 +26,7 @@ type Storage interface {
 	Close() error
 	StartBatch()
 	FinishBatch() error
+	CompactDB() error
 }
 
 type levelDB struct {
@@ -49,6 +52,25 @@ func OpenDB(path string) (Storage, error) {
 	return &levelDB{db: db}, nil
 }
 
+// RecoverDB recovers LevelDB database from corruption
+func RecoverDB(path string) error {
+	stor, err := storage.OpenFile(path)
+	if err != nil {
+		return err
+	}
+
+	db, err := leveldb.Recover(stor, nil)
+	if err != nil {
+		return err
+	}
+
+	db.Close()
+	stor.Close()
+
+	return nil
+}
+
+// Get key value from database
 func (l *levelDB) Get(key []byte) ([]byte, error) {
 	value, err := l.db.Get(key, nil)
 	if err != nil {
@@ -61,6 +83,7 @@ func (l *levelDB) Get(key []byte) ([]byte, error) {
 	return value, nil
 }
 
+// Put saves key to database, if key has the same value in DB already, it is not saved
 func (l *levelDB) Put(key []byte, value []byte) error {
 	if l.batch != nil {
 		l.batch.Put(key, value)
@@ -79,6 +102,7 @@ func (l *levelDB) Put(key []byte, value []byte) error {
 	return l.db.Put(key, value, nil)
 }
 
+// Delete removes key from DB
 func (l *levelDB) Delete(key []byte) error {
 	if l.batch != nil {
 		l.batch.Delete(key)
@@ -87,6 +111,7 @@ func (l *levelDB) Delete(key []byte) error {
 	return l.db.Delete(key, nil)
 }
 
+// KeysByPrefix returns all keys that start with prefix
 func (l *levelDB) KeysByPrefix(prefix []byte) [][]byte {
 	result := make([][]byte, 0, 20)
 
@@ -103,6 +128,7 @@ func (l *levelDB) KeysByPrefix(prefix []byte) [][]byte {
 	return result
 }
 
+// FetchByPrefix returns all values with keys that start with prefix
 func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
 	result := make([][]byte, 0, 20)
 
@@ -119,10 +145,14 @@ func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
 	return result
 }
 
+// Close finishes DB work
 func (l *levelDB) Close() error {
 	return l.db.Close()
 }
 
+// StartBatch starts batch processing of keys
+//
+// All subsequent Get, Put and Delete would work on batch
 func (l *levelDB) StartBatch() {
 	if l.batch != nil {
 		panic("batch already started")
@@ -130,6 +160,7 @@ func (l *levelDB) StartBatch() {
 	l.batch = new(leveldb.Batch)
 }
 
+// FinishBatch finalizes the batch, saving operations
 func (l *levelDB) FinishBatch() error {
 	if l.batch == nil {
 		panic("no batch")
@@ -138,3 +169,8 @@ func (l *levelDB) FinishBatch() error {
 	l.batch = nil
 	return err
 }
+
+// CompactDB compacts database by merging layers
+func (l *levelDB) CompactDB() error {
+	return l.db.CompactRange(util.Range{})
+}

database/leveldb_test.go

@@ -11,7 +11,8 @@ func Test(t *testing.T) {
 }
 
 type LevelDBSuite struct {
-	db Storage
+	path string
+	db   Storage
 }
 
 var _ = Suite(&LevelDBSuite{})
@@ -19,7 +20,8 @@ var _ = Suite(&LevelDBSuite{})
 func (s *LevelDBSuite) SetUpTest(c *C) {
 	var err error
 
-	s.db, err = OpenDB(c.MkDir())
+	s.path = c.MkDir()
+	s.db, err = OpenDB(s.path)
 	c.Assert(err, IsNil)
 }
 
@@ -28,6 +30,29 @@ func (s *LevelDBSuite) TearDownTest(c *C) {
 	c.Assert(err, IsNil)
 }
 
+func (s *LevelDBSuite) TestRecoverDB(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Check(err, IsNil)
+
+	err = s.db.Close()
+	c.Check(err, IsNil)
+
+	err = RecoverDB(s.path)
+	c.Check(err, IsNil)
+
+	s.db, err = OpenDB(s.path)
+	c.Check(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}
+
 func (s *LevelDBSuite) TestGetPut(c *C) {
 	var (
 		key   = []byte("key")
@@ -122,3 +147,11 @@ func (s *LevelDBSuite) TestBatch(c *C) {
 	s.db.StartBatch()
 	c.Check(func() { s.db.StartBatch() }, Panics, "batch already started")
 }
+
+func (s *LevelDBSuite) TestCompactDB(c *C) {
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+
+	c.Check(s.db.CompactDB(), IsNil)
+}

deb/collections.go

@@ -0,0 +1,65 @@
+package deb
+
+import (
+	"github.com/smira/aptly/database"
+)
+
+// CollectionFactory is a single place to generate all desired collections
+type CollectionFactory struct {
+	db             database.Storage
+	packages       *PackageCollection
+	remoteRepos    *RemoteRepoCollection
+	snapshots      *SnapshotCollection
+	localRepos     *LocalRepoCollection
+	publishedRepos *PublishedRepoCollection
+}
+
+// NewCollectionFactory creates new factory
+func NewCollectionFactory(db database.Storage) *CollectionFactory {
+	return &CollectionFactory{db: db}
+}
+
+// PackageCollection returns (or creates) new PackageCollection
+func (factory *CollectionFactory) PackageCollection() *PackageCollection {
+	if factory.packages == nil {
+		factory.packages = NewPackageCollection(factory.db)
+	}
+
+	return factory.packages
+}
+
+// RemoteRepoCollection returns (or creates) new RemoteRepoCollection
+func (factory *CollectionFactory) RemoteRepoCollection() *RemoteRepoCollection {
+	if factory.remoteRepos == nil {
+		factory.remoteRepos = NewRemoteRepoCollection(factory.db)
+	}
+
+	return factory.remoteRepos
+}
+
+// SnapshotCollection returns (or creates) new SnapshotCollection
+func (factory *CollectionFactory) SnapshotCollection() *SnapshotCollection {
+	if factory.snapshots == nil {
+		factory.snapshots = NewSnapshotCollection(factory.db)
+	}
+
+	return factory.snapshots
+}
+
+// LocalRepoCollection returns (or creates) new LocalRepoCollection
+func (factory *CollectionFactory) LocalRepoCollection() *LocalRepoCollection {
+	if factory.localRepos == nil {
+		factory.localRepos = NewLocalRepoCollection(factory.db)
+	}
+
+	return factory.localRepos
+}
+
+// PublishedRepoCollection returns (or creates) new PublishedRepoCollection
+func (factory *CollectionFactory) PublishedRepoCollection() *PublishedRepoCollection {
+	if factory.publishedRepos == nil {
+		factory.publishedRepos = NewPublishedRepoCollection(factory.db)
+	}
+
+	return factory.publishedRepos
+}

deb/deb.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"archive/tar"

deb/deb_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"github.com/smira/aptly/utils"

deb/debian.go

@@ -0,0 +1,2 @@
+// Package deb implements Debian-specific repository handling
+package deb

deb/debian_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	. "launchpad.net/gocheck"

deb/format.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bufio"
@@ -11,8 +11,9 @@ import (
 type Stanza map[string]string
 
 // Canonical order of fields in stanza
-var canocialOrder = []string{"Origin", "Label", "Suite", "Package", "Version", "Installed-Size", "Priority", "Section", "Maintainer",
-	"Architecture", "Codename", "Date", "Architectures", "Components", "Description", "MD5sum", "MD5Sum", "SHA1", "SHA256"}
+var canocialOrder = []string{"Package", "Origin", "Label", "Suite", "Version", "Installed-Size", "Priority", "Section", "Maintainer",
+	"Architecture", "Codename", "Date", "Architectures", "Components", "Description", "MD5sum", "MD5Sum", "SHA1", "SHA256",
+	"Archive", "Component"}
 
 // Copy returns copy of Stanza
 func (s Stanza) Copy() (result Stanza) {

deb/format_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bufio"

deb/list.go

@@ -1,11 +1,10 @@
-package debian
+package deb
 
 import (
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
 	"sort"
-	"strings"
 )
 
 // Dependency options
@@ -63,9 +62,9 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 	}
 
 	err := reflist.ForEach(func(key []byte) error {
-		p, err := collection.ByKey(key)
-		if err != nil {
-			return fmt.Errorf("unable to load package with key %s: %s", key, err)
+		p, err2 := collection.ByKey(key)
+		if err2 != nil {
+			return fmt.Errorf("unable to load package with key %s: %s", key, err2)
 		}
 		if progress != nil {
 			progress.AddBar(1)
@@ -86,11 +85,11 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 
 // Add appends package to package list, additionally checking for uniqueness
 func (l *PackageList) Add(p *Package) error {
-	key := string(p.Key(""))
+	key := string(p.ShortKey(""))
 	existing, ok := l.packages[key]
 	if ok {
 		if !existing.Equals(p) {
-			return fmt.Errorf("conflict in package %s: %#v != %#v", p, existing, p)
+			return fmt.Errorf("conflict in package %s", p)
 		}
 		return nil
 	}
@@ -101,7 +100,7 @@ func (l *PackageList) Add(p *Package) error {
 			l.providesIndex[provides] = append(l.providesIndex[provides], p)
 		}
 
-		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= p.Name })
+		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.lessPackages(p, l.packagesIndex[j]) })
 
 		// insert p into l.packagesIndex in position i
 		l.packagesIndex = append(l.packagesIndex, nil)
@@ -123,6 +122,22 @@ func (l *PackageList) ForEach(handler func(*Package) error) error {
 	return err
 }
 
+// ForEachIndexed calls handler for each package in list in indexed order
+func (l *PackageList) ForEachIndexed(handler func(*Package) error) error {
+	if !l.indexed {
+		panic("list not indexed, can't iterate")
+	}
+
+	var err error
+	for _, p := range l.packagesIndex {
+		err = handler(p)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
 // Len returns number of packages in the list
 func (l *PackageList) Len() int {
 	return len(l.packages)
@@ -137,7 +152,7 @@ func (l *PackageList) Append(pl *PackageList) error {
 		existing, ok := l.packages[k]
 		if ok {
 			if !existing.Equals(p) {
-				return fmt.Errorf("conflict in package %s: %#v != %#v", p, existing, p)
+				return fmt.Errorf("conflict in package %s", p)
 			}
 		} else {
 			l.packages[k] = p
@@ -149,7 +164,7 @@ func (l *PackageList) Append(pl *PackageList) error {
 
 // Remove removes package from the list, and updates index when required
 func (l *PackageList) Remove(p *Package) {
-	delete(l.packages, string(p.Key("")))
+	delete(l.packages, string(p.ShortKey("")))
 	if l.indexed {
 		for _, provides := range p.Provides {
 			for i, pkg := range l.providesIndex[provides] {
@@ -263,7 +278,7 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 						continue
 					}
 
-					if sources.Search(dep) == nil {
+					if sources.Search(dep, false) == nil {
 						variantsMissing = append(variantsMissing, dep)
 						missingCount++
 					} else {
@@ -300,9 +315,21 @@ func (l *PackageList) Swap(i, j int) {
 	l.packagesIndex[i], l.packagesIndex[j] = l.packagesIndex[j], l.packagesIndex[i]
 }
 
-// Compare compares two names in lexographical order
+func (l *PackageList) lessPackages(iPkg, jPkg *Package) bool {
+	if iPkg.Name == jPkg.Name {
+		cmp := CompareVersions(iPkg.Version, jPkg.Version)
+		if cmp == 0 {
+			return iPkg.Architecture < jPkg.Architecture
+		}
+		return cmp == 1
+	}
+
+	return iPkg.Name < jPkg.Name
+}
+
+// Less compares two packages by name (lexographical) and version (latest to oldest)
 func (l *PackageList) Less(i, j int) bool {
-	return l.packagesIndex[i].Name < l.packagesIndex[j].Name
+	return l.lessPackages(l.packagesIndex[i], l.packagesIndex[j])
 }
 
 // PrepareIndex prepares list for indexing
@@ -325,16 +352,32 @@ func (l *PackageList) PrepareIndex() {
 	l.indexed = true
 }
 
-// Search searches package index for specified package
-func (l *PackageList) Search(dep Dependency) *Package {
+// Scan searches package index using full scan
+func (l *PackageList) Scan(q PackageQuery) (result *PackageList) {
+	result = NewPackageList()
+	for _, pkg := range l.packages {
+		if q.Matches(pkg) {
+			result.Add(pkg)
+		}
+	}
+
+	return
+}
+
+// Search searches package index for specified package(s) using optimized queries
+func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*Package) {
 	if !l.indexed {
 		panic("list not indexed, can't search")
 	}
 
 	if dep.Relation == VersionDontCare {
 		for _, p := range l.providesIndex[dep.Pkg] {
-			if p.MatchesArchitecture(dep.Architecture) {
-				return p
+			if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
+				searchResults = append(searchResults, p)
+
+				if !allMatches {
+					break
+				}
 			}
 		}
 	}
@@ -344,16 +387,21 @@ func (l *PackageList) Search(dep Dependency) *Package {
 	for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
 		p := l.packagesIndex[i]
 		if p.MatchesDependency(dep) {
-			return p
+			searchResults = append(searchResults, p)
+
+			if !allMatches {
+				break
+			}
 		}
 
 		i++
 	}
-	return nil
+
+	return
 }
 
 // Filter filters package index by specified queries (ORed together), possibly pulling dependencies
-func (l *PackageList) Filter(queries []string, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
+func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
 	if !l.indexed {
 		panic("list not indexed, can't filter")
 	}
@@ -361,53 +409,16 @@ func (l *PackageList) Filter(queries []string, withDependencies bool, source *Pa
 	result := NewPackageList()
 
 	for _, query := range queries {
-		isDepQuery := strings.IndexAny(query, " (){}=<>") != -1
-
-		if !isDepQuery {
-			// try to interpret query as package string representation
-
-			// convert Package.String() to Package.Key()
-			i := strings.Index(query, "_")
-			if i != -1 {
-				pkg, query := query[:i], query[i+1:]
-				j := strings.LastIndex(query, "_")
-				if j != -1 {
-					version, arch := query[:j], query[j+1:]
-					p := l.packages["P"+arch+" "+pkg+" "+version]
-					if p != nil {
-						result.Add(p)
-						continue
-					}
-				}
-			}
-		}
-
-		// try as dependency
-		dep, err := ParseDependency(query)
-		if err != nil {
-			if isDepQuery {
-				return nil, err
-			}
-			// parsing failed, but probably that wasn't a dep query
-			continue
-		}
-
-		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= dep.Pkg })
-
-		for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
-			p := l.packagesIndex[i]
-			if p.MatchesDependency(dep) {
-				result.Add(p)
-			}
-			i++
-		}
+		result.Append(query.Query(l))
 	}
 
 	if withDependencies {
 		added := result.Len()
 
 		dependencySource := NewPackageList()
-		dependencySource.Append(source)
+		if source != nil {
+			dependencySource.Append(source)
+		}
 		dependencySource.Append(result)
 		dependencySource.PrepareIndex()
 
@@ -423,11 +434,13 @@ func (l *PackageList) Filter(queries []string, withDependencies bool, source *Pa
 
 			// try to satisfy dependencies
 			for _, dep := range missing {
-				p := l.Search(dep)
-				if p != nil {
-					result.Add(p)
-					dependencySource.Add(p)
-					added++
+				searchResults := l.Search(dep, false)
+				if searchResults != nil {
+					for _, p := range searchResults {
+						result.Add(p)
+						dependencySource.Add(p)
+						added++
+					}
 				}
 			}
 		}

deb/list_test.go

@@ -1,12 +1,49 @@
-package debian
+package deb
 
 import (
 	"errors"
 	. "launchpad.net/gocheck"
+	"regexp"
 	"sort"
 	"strings"
 )
 
+type containsChecker struct {
+	*CheckerInfo
+}
+
+func (c *containsChecker) Check(params []interface{}, names []string) (result bool, error string) {
+	var (
+		pkgSlice1 []*Package
+		pkgSlice2 []*Package
+		ok        bool
+	)
+
+	pkgMap := make(map[*Package]bool)
+
+	pkgSlice1, ok = params[0].([]*Package)
+	if !ok {
+		return false, "The first parameter is not a Package slice"
+	}
+	pkgSlice2, ok = params[1].([]*Package)
+	if !ok {
+		return false, "The second parameter is not a Package slice"
+	}
+
+	for _, pkg := range pkgSlice2 {
+		pkgMap[pkg] = true
+	}
+
+	for _, pkg := range pkgSlice1 {
+		if _, ok := pkgMap[pkg]; !ok {
+			return false, ""
+		}
+	}
+	return true, ""
+}
+
+var Contains = &containsChecker{&CheckerInfo{Name: "Contains", Params: []string{"Container", "Expected to contain"}}}
+
 type PackageListSuite struct {
 	// Simple list with "real" packages from stanzas
 	list                   *PackageList
@@ -14,8 +51,10 @@ type PackageListSuite struct {
 
 	// Mocked packages in list
 	packages       []*Package
+	packages2      []*Package
 	sourcePackages []*Package
 	il             *PackageList
+	il2            *PackageList
 }
 
 var _ = Suite(&PackageListSuite{})
@@ -29,7 +68,7 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 	stanza["Package"] = "mars-invaders"
 	s.p3 = NewPackageFromControlFile(stanza)
 	stanza = packageStanza.Copy()
-	stanza["Size"] = "42"
+	stanza["Source"] = "unknown-planet"
 	s.p4 = NewPackageFromControlFile(stanza)
 	stanza = packageStanza.Copy()
 	stanza["Package"] = "lonely-strangers"
@@ -60,6 +99,20 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 	}
 	s.il.PrepareIndex()
 
+	s.il2 = NewPackageList()
+	s.packages2 = []*Package{
+		&Package{Name: "mailer", Version: "3.5.8", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		&Package{Name: "sendmail", Version: "1.0", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		&Package{Name: "app", Version: "1.1-bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		&Package{Name: "app", Version: "1.1-bp2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		&Package{Name: "app", Version: "1.2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
+		&Package{Name: "app", Version: "3.0", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+	}
+	for _, p := range s.packages2 {
+		s.il2.Add(p)
+	}
+	s.il2.PrepareIndex()
+
 	s.sourcePackages = []*Package{
 		&Package{Name: "postfix", Version: "1.3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
 		&Package{Name: "app", Version: "1.1~bp1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
@@ -196,35 +249,60 @@ func (s *PackageListSuite) TestAppend(c *C) {
 }
 
 func (s *PackageListSuite) TestSearch(c *C) {
-	c.Check(func() { s.list.Search(Dependency{Architecture: "i386", Pkg: "app"}) }, Panics, "list not indexed, can't search")
+	//allMatches = False
+	c.Check(func() { s.list.Search(Dependency{Architecture: "i386", Pkg: "app"}, false) }, Panics, "list not indexed, can't search")
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "mail-agent"}), Equals, s.packages[4])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "puppy"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "mail-agent"}, false), DeepEquals, []*Package{s.packages[4]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "puppy"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp2"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp2"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1~~"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1~~"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~bp1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~~"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~~"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.0"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.2"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.0"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.2"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}, false), IsNil)
+
+	// search w/o version should return package with latest version
+	c.Check(s.il.Search(Dependency{Architecture: "source", Pkg: "dpkg"}, false), DeepEquals, []*Package{s.packages[13]})
+
+	// allMatches = True
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "mail-agent"}, true), Contains, []*Package{s.packages2[0], s.packages2[1]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "puppy"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "3"}, true), Contains, []*Package{s.packages2[5]})
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLess, Version: "1.2"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLess, Version: "1.1~"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.2"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1-bp1"}, true), Contains, []*Package{s.packages2[2]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.0"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreater, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreater, Version: "5.0"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}, true), Contains, []*Package{s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "5.0"}, true), IsNil)
 }
 
 func (s *PackageListSuite) TestFilter(c *C) {
-	c.Check(func() { s.list.Filter([]string{"abcd_0.3_i386"}, false, nil, 0, nil) }, Panics, "list not indexed, can't filter")
-
-	_, err := s.il.Filter([]string{"app >3)"}, false, nil, 0, nil)
-	c.Check(err, ErrorMatches, "unable to parse dependency.*")
+	c.Check(func() { s.list.Filter([]PackageQuery{&PkgQuery{"abcd", "0.3", "i386"}}, false, nil, 0, nil) }, Panics, "list not indexed, can't filter")
 
 	plString := func(l *PackageList) string {
 		list := make([]string, 0, l.Len())
@@ -237,25 +315,69 @@ func (s *PackageListSuite) TestFilter(c *C) {
 		return strings.Join(list, " ")
 	}
 
-	result, err := s.il.Filter([]string{"app_1.1~bp1_i386"}, false, nil, 0, nil)
+	result, err := s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
 
-	result, err = s.il.Filter([]string{"app_1.1~bp1_i386", "dpkg_1.7_source", "dpkg_1.8_amd64"}, false, nil, 0, nil)
+	result, err = s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, &PkgQuery{"dpkg", "1.7", "source"},
+		&PkgQuery{"dpkg", "1.8", "amd64"}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_source")
 
-	result, err = s.il.Filter([]string{"app", "dpkg (>>1.6.1-3)", "app (>=1.0)", "xyz", "aa (>>3.0)"}, false, nil, 0, nil)
+	result, err = s.il.Filter([]PackageQuery{
+		&DependencyQuery{Dep: Dependency{Pkg: "app"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "xyz"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "aa", Relation: VersionGreater, Version: "3.0"}}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source")
 
-	result, err = s.il.Filter([]string{"app {i386}"}, true, NewPackageList(), 0, []string{"i386"})
+	result, err = s.il.Filter([]PackageQuery{&DependencyQuery{Dep: Dependency{Pkg: "app", Architecture: "i386"}}}, true, NewPackageList(), 0, []string{"i386"})
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386")
 
-	result, err = s.il.Filter([]string{"app (>=0.9)", "lib", "data"}, true, NewPackageList(), 0, []string{"i386", "amd64"})
+	result, err = s.il.Filter([]PackageQuery{
+		&DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "0.9"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "lib"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "data"}}}, true, NewPackageList(), 0, []string{"i386", "amd64"})
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "")
+
+	result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_i386 data_1.1~bp1_all")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{&FieldQuery{Field: "Version", Relation: VersionGreaterOrEqual, Value: "1.0"},
+		&FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 data_1.1~bp1_all")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&NotQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.6.1-3_arm dpkg_1.6.1-3_source dpkg_1.7_source libx_1.5_arm")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionRegexp, Value: "i.*6", Regexp: regexp.MustCompile("i.*6")}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
 }
 
 func (s *PackageListSuite) TestVerifyDependencies(c *C) {

deb/local.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -17,6 +17,10 @@ type LocalRepo struct {
 	Name string
 	// Comment
 	Comment string
+	// DefaultDistribution
+	DefaultDistribution string `codec:",omitempty"`
+	// DefaultComponent
+	DefaultComponent string `codec:",omitempty"`
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 }

deb/local_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"errors"

deb/package.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"fmt"
@@ -26,6 +26,8 @@ type Package struct {
 	IsSource bool
 	// Hash of files section
 	FilesHash uint64
+	// Is this >= 0.6 package?
+	V06Plus bool
 	// Offload fields
 	deps  *PackageDependencies
 	extra *Stanza
@@ -41,6 +43,7 @@ func NewPackageFromControlFile(input Stanza) *Package {
 		Version:      input["Version"],
 		Architecture: input["Architecture"],
 		Source:       input["Source"],
+		V06Plus:      true,
 	}
 
 	delete(input, "Package")
@@ -89,6 +92,7 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 		Version:            input["Version"],
 		Architecture:       "source",
 		SourceArchitecture: input["Architecture"],
+		V06Plus:            true,
 	}
 
 	delete(input, "Package")
@@ -167,7 +171,16 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 
 // Key returns unique key identifying package
 func (p *Package) Key(prefix string) []byte {
-	return []byte(prefix + "P" + p.Architecture + " " + p.Name + " " + p.Version)
+	if p.V06Plus {
+		return []byte(fmt.Sprintf("%sP%s %s %s %08x", prefix, p.Architecture, p.Name, p.Version, p.FilesHash))
+	}
+
+	return p.ShortKey(prefix)
+}
+
+// ShortKey returns key for the package that should be unique in one list
+func (p *Package) ShortKey(prefix string) []byte {
+	return []byte(fmt.Sprintf("%sP%s %s %s", prefix, p.Architecture, p.Name, p.Version))
 }
 
 // String creates readable representation
@@ -175,6 +188,70 @@ func (p *Package) String() string {
 	return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.Architecture)
 }
 
+// GetField returns fields from package
+func (p *Package) GetField(name string) string {
+	switch name {
+	// $Version is handled in FieldQuery
+	case "$Source":
+		if p.IsSource {
+			return ""
+		}
+		source := p.Source
+		if source == "" {
+			return p.Name
+		} else if pos := strings.Index(source, "("); pos != -1 {
+			return strings.TrimSpace(source[:pos])
+		}
+		return source
+	case "$SourceVersion":
+		if p.IsSource {
+			return ""
+		}
+		source := p.Source
+		if pos := strings.Index(source, "("); pos != -1 {
+			if pos2 := strings.LastIndex(source, ")"); pos2 != -1 && pos2 > pos {
+				return strings.TrimSpace(source[pos+1 : pos2])
+			}
+		}
+		return p.Version
+	case "$Architecture":
+		return p.Architecture
+	case "$PackageType":
+		if p.IsSource {
+			return "source"
+		}
+		return "deb"
+	case "Name":
+		return p.Name
+	case "Version":
+		return p.Version
+	case "Architecture":
+		if p.IsSource {
+			return p.SourceArchitecture
+		}
+		return p.Architecture
+	case "Source":
+		return p.Source
+	case "Depends":
+		return strings.Join(p.Deps().Depends, ", ")
+	case "Pre-Depends":
+		return strings.Join(p.Deps().PreDepends, ", ")
+	case "Suggests":
+		return strings.Join(p.Deps().Suggests, ", ")
+	case "Recommends":
+		return strings.Join(p.Deps().Recommends, ", ")
+	case "Provides":
+		return strings.Join(p.Provides, ", ")
+	case "Build-Depends":
+		return strings.Join(p.Deps().BuildDepends, ", ")
+	case "Build-Depends-Indep":
+		return strings.Join(p.Deps().BuildDependsInDep, ", ")
+	default:
+		return p.Extra()[name]
+	}
+	return ""
+}
+
 // MatchesArchitecture checks whether packages matches specified architecture
 func (p *Package) MatchesArchitecture(arch string) bool {
 	if p.Architecture == "all" && arch != "source" {
@@ -186,19 +263,23 @@ func (p *Package) MatchesArchitecture(arch string) bool {
 
 // MatchesDependency checks whether package matches specified dependency
 func (p *Package) MatchesDependency(dep Dependency) bool {
-	if dep.Pkg != p.Name {
-		return false
-	}
-
 	if dep.Architecture != "" && !p.MatchesArchitecture(dep.Architecture) {
 		return false
 	}
 
 	if dep.Relation == VersionDontCare {
-		return true
+		if utils.StrSliceHasItem(p.Provides, dep.Pkg) {
+			return true
+		}
+		return dep.Pkg == p.Name
+	}
+
+	if dep.Pkg != p.Name {
+		return false
 	}
 
 	r := CompareVersions(p.Version, dep.Version)
+
 	switch dep.Relation {
 	case VersionEqual:
 		return r == 0
@@ -210,6 +291,11 @@ func (p *Package) MatchesDependency(dep Dependency) bool {
 		return r <= 0
 	case VersionGreaterOrEqual:
 		return r >= 0
+	case VersionPatternMatch:
+		matched, err := filepath.Match(dep.Version, p.Version)
+		return err == nil && matched
+	case VersionRegexp:
+		return dep.Regexp.FindStringIndex(p.Version) != nil
 	}
 
 	panic("unknown relation")
@@ -388,16 +474,18 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 			return err
 		}
 
-		relPath, err := publishedStorage.LinkFromPool(prefix, component, poolDir, packagePool, sourcePath)
+		relPath := filepath.Join("pool", component, poolDir)
+		publishedDirectory := filepath.Join(prefix, relPath)
+
+		err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePath, f.Checksums.MD5)
 		if err != nil {
 			return err
 		}
 
-		dir := filepath.Dir(relPath)
 		if p.IsSource {
-			p.Extra()["Directory"] = dir
+			p.Extra()["Directory"] = relPath
 		} else {
-			p.Files()[i].downloadPath = dir
+			p.Files()[i].downloadPath = relPath
 		}
 	}
 
@@ -409,6 +497,8 @@ func (p *Package) PoolDirectory() (string, error) {
 	source := p.Source
 	if source == "" {
 		source = p.Name
+	} else if pos := strings.Index(source, "("); pos != -1 {
+		source = strings.TrimSpace(source[:pos])
 	}
 
 	if len(source) < 2 {

deb/package_collection.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -12,12 +12,14 @@ import (
 type PackageCollection struct {
 	db           database.Storage
 	encodeBuffer bytes.Buffer
+	codecHandle  *codec.MsgpackHandle
 }
 
 // NewPackageCollection creates new PackageCollection and binds it to database
 func NewPackageCollection(db database.Storage) *PackageCollection {
 	return &PackageCollection{
-		db: db,
+		db:          db,
+		codecHandle: &codec.MsgpackHandle{},
 	}
 }
 
@@ -53,7 +55,7 @@ func (collection *PackageCollection) ByKey(key []byte) (*Package, error) {
 	if len(encoded) > 2 && (encoded[0] != 0xc1 || encoded[1] != 0x1) {
 		oldp := &oldPackage{}
 
-		decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+		decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 		err = decoder.Decode(oldp)
 		if err != nil {
 			return nil, err
@@ -83,12 +85,12 @@ func (collection *PackageCollection) ByKey(key []byte) (*Package, error) {
 		p.UpdateFiles(PackageFiles(oldp.Files))
 
 		// Save in new format
-		err = collection.internalUpdate(p)
+		err = collection.Update(p)
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		decoder := codec.NewDecoderBytes(encoded[2:], &codec.MsgpackHandle{})
+		decoder := codec.NewDecoderBytes(encoded[2:], collection.codecHandle)
 		err = decoder.Decode(p)
 		if err != nil {
 			return nil, err
@@ -109,7 +111,7 @@ func (collection *PackageCollection) loadExtra(p *Package) *Stanza {
 
 	stanza := &Stanza{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(stanza)
 	if err != nil {
 		panic("unable to decode extra")
@@ -127,7 +129,7 @@ func (collection *PackageCollection) loadDependencies(p *Package) *PackageDepend
 
 	deps := &PackageDependencies{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(deps)
 	if err != nil {
 		panic("unable to decode deps")
@@ -145,7 +147,7 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 
 	files := &PackageFiles{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(files)
 	if err != nil {
 		panic("unable to decode files")
@@ -156,26 +158,7 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 
 // Update adds or updates information about package in DB checking for conficts first
 func (collection *PackageCollection) Update(p *Package) error {
-	existing, err := collection.ByKey(p.Key(""))
-	if err == nil {
-		// if .Files is different, consider to be conflict
-		if p.FilesHash != existing.FilesHash {
-			return fmt.Errorf("unable to save: %s, conflict with existing packge", p)
-		}
-		// ok, .Files are the same, but maybe some meta-data is different, proceed to saving
-	} else {
-		if err != database.ErrNotFound {
-			return err
-		}
-		// ok, package doesn't exist yet
-	}
-
-	return collection.internalUpdate(p)
-}
-
-// internalUpdate updates information in DB about package and offloaded fields
-func (collection *PackageCollection) internalUpdate(p *Package) error {
-	encoder := codec.NewEncoder(&collection.encodeBuffer, &codec.MsgpackHandle{})
+	encoder := codec.NewEncoder(&collection.encodeBuffer, collection.codecHandle)
 
 	collection.encodeBuffer.Reset()
 	collection.encodeBuffer.WriteByte(0xc1)

deb/package_collection_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"github.com/smira/aptly/database"
@@ -48,20 +48,6 @@ func (s *PackageCollectionSuite) TestUpdate(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(res.Equals(s.p), Equals, false)
 	c.Assert(res.Equals(p2), Equals, true)
-
-	// change file info
-	p2 = NewPackageFromControlFile(packageStanza.Copy())
-	p2.UpdateFiles(nil)
-	res, err = s.collection.ByKey(p2.Key(""))
-	err = s.collection.Update(p2)
-	c.Assert(err, ErrorMatches, ".*conflict with existing packge")
-	p2 = NewPackageFromControlFile(packageStanza.Copy())
-	files := p2.Files()
-	files[0].Checksums.MD5 = "abcdef"
-	p2.UpdateFiles(files)
-	res, err = s.collection.ByKey(p2.Key(""))
-	err = s.collection.Update(p2)
-	c.Assert(err, ErrorMatches, ".*conflict with existing packge")
 }
 
 func (s *PackageCollectionSuite) TestByKey(c *C) {

deb/package_deps.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"strings"

deb/package_files.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"encoding/binary"

deb/package_files_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"github.com/smira/aptly/files"

deb/package_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -7,6 +7,7 @@ import (
 	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
+	"regexp"
 )
 
 type PackageSuite struct {
@@ -87,10 +88,21 @@ func (s *PackageSuite) TestWithProvides(c *C) {
 func (s *PackageSuite) TestKey(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
+	c.Check(p.Key(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2 c8901eedd79ac51b"))
+	c.Check(p.Key("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2 c8901eedd79ac51b"))
+
+	p.V06Plus = false
 	c.Check(p.Key(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2"))
 	c.Check(p.Key("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2"))
 }
 
+func (s *PackageSuite) TestShortKey(c *C) {
+	p := NewPackageFromControlFile(s.stanza)
+
+	c.Check(p.ShortKey(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2"))
+	c.Check(p.ShortKey("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2"))
+}
+
 func (s *PackageSuite) TestStanza(c *C) {
 	p := NewPackageFromControlFile(s.stanza.Copy())
 	stanza := p.Stanza()
@@ -108,6 +120,58 @@ func (s *PackageSuite) TestString(c *C) {
 	c.Assert(p.String(), Equals, "alien-arena-common_7.40-2_i386")
 }
 
+func (s *PackageSuite) TestGetField(c *C) {
+	p := NewPackageFromControlFile(s.stanza.Copy())
+
+	stanza2 := s.stanza.Copy()
+	delete(stanza2, "Source")
+	stanza2["Provides"] = "app, game"
+	p2 := NewPackageFromControlFile(stanza2)
+
+	stanza3 := s.stanza.Copy()
+	stanza3["Source"] = "alien-arena (3.5)"
+	p3 := NewPackageFromControlFile(stanza3)
+
+	p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
+
+	c.Check(p.GetField("$Source"), Equals, "alien-arena")
+	c.Check(p2.GetField("$Source"), Equals, "alien-arena-common")
+	c.Check(p3.GetField("$Source"), Equals, "alien-arena")
+	c.Check(p4.GetField("$Source"), Equals, "")
+
+	c.Check(p.GetField("$SourceVersion"), Equals, "7.40-2")
+	c.Check(p2.GetField("$SourceVersion"), Equals, "7.40-2")
+	c.Check(p3.GetField("$SourceVersion"), Equals, "3.5")
+	c.Check(p4.GetField("$SourceVersion"), Equals, "")
+
+	c.Check(p.GetField("$Architecture"), Equals, "i386")
+	c.Check(p4.GetField("$Architecture"), Equals, "source")
+
+	c.Check(p.GetField("$PackageType"), Equals, "deb")
+	c.Check(p4.GetField("$PackageType"), Equals, "source")
+
+	c.Check(p.GetField("Name"), Equals, "alien-arena-common")
+	c.Check(p4.GetField("Name"), Equals, "access-modifier-checker")
+
+	c.Check(p.GetField("Architecture"), Equals, "i386")
+	c.Check(p4.GetField("Architecture"), Equals, "all")
+
+	c.Check(p.GetField("Version"), Equals, "7.40-2")
+
+	c.Check(p.GetField("Source"), Equals, "alien-arena")
+	c.Check(p2.GetField("Source"), Equals, "")
+	c.Check(p3.GetField("Source"), Equals, "alien-arena (3.5)")
+	c.Check(p4.GetField("Source"), Equals, "")
+
+	c.Check(p.GetField("Depends"), Equals, "libc6 (>= 2.7), alien-arena-data (>= 7.40)")
+
+	c.Check(p.GetField("Provides"), Equals, "")
+	c.Check(p2.GetField("Provides"), Equals, "app, game")
+
+	c.Check(p.GetField("Section"), Equals, "contrib/games")
+	c.Check(p.GetField("Priority"), Equals, "extra")
+}
+
 func (s *PackageSuite) TestEquals(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
@@ -163,6 +227,9 @@ func (s *PackageSuite) TestMatchesDependency(c *C) {
 	// exact match
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionEqual, Version: "7.40-2"}), Equals, true)
 
+	// exact match, same version, no revision specified
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionEqual, Version: "7.40"}), Equals, false)
+
 	// different name
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena", Architecture: "i386", Relation: VersionEqual, Version: "7.40-2"}), Equals, false)
 
@@ -193,6 +260,29 @@ func (s *PackageSuite) TestMatchesDependency(c *C) {
 	// <=
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionLessOrEqual, Version: "7.40-2"}), Equals, true)
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionLessOrEqual, Version: "7.40-1"}), Equals, false)
+
+	// %
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-*"}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-[2]"}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-[2"}), Equals, false)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-[34]"}), Equals, false)
+
+	// ~
+	c.Check(
+		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
+			Regexp: regexp.MustCompile("7\\.40-.*")}), Equals, true)
+	c.Check(
+		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
+			Regexp: regexp.MustCompile("40")}), Equals, true)
+	c.Check(
+		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
+			Regexp: regexp.MustCompile("39-.*")}), Equals, false)
+
+	// Provides
+	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Relation: VersionDontCare}), Equals, false)
+	p.Provides = []string{"fun", "game"}
+	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Relation: VersionDontCare}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Architecture: "amd64", Relation: VersionDontCare}), Equals, false)
 }
 
 func (s *PackageSuite) TestGetDependencies(c *C) {
@@ -230,6 +320,12 @@ func (s *PackageSuite) TestPoolDirectory(c *C) {
 	c.Check(err, IsNil)
 	c.Check(dir, Equals, "liba/libarena")
 
+	p = NewPackageFromControlFile(packageStanza.Copy())
+	p.Source = "gcc-defaults (1.77)"
+	dir, err = p.PoolDirectory()
+	c.Check(err, IsNil)
+	c.Check(dir, Equals, "g/gcc-defaults")
+
 	p = NewPackageFromControlFile(packageStanza.Copy())
 	p.Source = "l"
 	_, err = p.PoolDirectory()

deb/ppa.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"fmt"
@@ -11,14 +11,14 @@ import (
 var ppaRegexp = regexp.MustCompile("^ppa:([^/]+)/(.+)$")
 
 // ParsePPA converts ppa URL like ppa:user/ppa-name to full HTTP url
-func ParsePPA(ppaURL string) (url string, distribution string, components []string, err error) {
+func ParsePPA(ppaURL string, config *utils.ConfigStructure) (url string, distribution string, components []string, err error) {
 	matches := ppaRegexp.FindStringSubmatch(ppaURL)
 	if matches == nil {
 		err = fmt.Errorf("unable to parse ppa URL: %v", ppaURL)
 		return
 	}
 
-	distributorID := utils.Config.PpaDistributorID
+	distributorID := config.PpaDistributorID
 	if distributorID == "" {
 		distributorID, err = getDistributorID()
 		if err != nil {
@@ -27,7 +27,7 @@ func ParsePPA(ppaURL string) (url string, distribution string, components []stri
 		}
 	}
 
-	codename := utils.Config.PpaCodename
+	codename := config.PpaCodename
 	if codename == "" {
 		codename, err = getCodename()
 		if err != nil {

deb/ppa_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"github.com/smira/aptly/utils"
@@ -6,27 +6,19 @@ import (
 )
 
 type PpaSuite struct {
-	savedConfig utils.ConfigStructure
+	config utils.ConfigStructure
 }
 
 var _ = Suite(&PpaSuite{})
 
-func (s *PpaSuite) SetUpTest(c *C) {
-	s.savedConfig = utils.Config
-}
-
-func (s *PpaSuite) TearDownTest(c *C) {
-	utils.Config = s.savedConfig
-}
-
 func (s *PpaSuite) TestParsePPA(c *C) {
-	_, _, _, err := ParsePPA("ppa:dedeed")
+	_, _, _, err := ParsePPA("ppa:dedeed", &s.config)
 	c.Check(err, ErrorMatches, "unable to parse ppa URL.*")
 
-	utils.Config.PpaDistributorID = "debian"
-	utils.Config.PpaCodename = "wheezy"
+	s.config.PpaDistributorID = "debian"
+	s.config.PpaCodename = "wheezy"
 
-	url, distribution, components, err := ParsePPA("ppa:user/project")
+	url, distribution, components, err := ParsePPA("ppa:user/project", &s.config)
 	c.Check(err, IsNil)
 	c.Check(url, Equals, "http://ppa.launchpad.net/user/project/debian")
 	c.Check(distribution, Equals, "wheezy")

deb/publish_test.go

@@ -0,0 +1,805 @@
+package deb
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/files"
+	"github.com/ugorji/go/codec"
+	"io/ioutil"
+	. "launchpad.net/gocheck"
+	"os"
+	"path/filepath"
+)
+
+type pathExistsChecker struct {
+	*CheckerInfo
+}
+
+var PathExists = &pathExistsChecker{
+	&CheckerInfo{Name: "PathExists", Params: []string{"path"}},
+}
+
+func (checker *pathExistsChecker) Check(params []interface{}, names []string) (result bool, error string) {
+	_, err := os.Stat(params[0].(string))
+	return err == nil, ""
+}
+
+type NullSigner struct{}
+
+func (n *NullSigner) Init() error {
+	return nil
+}
+
+func (n *NullSigner) SetKey(keyRef string) {
+}
+
+func (n *NullSigner) SetKeyRing(keyring, secretKeyring string) {
+}
+
+func (n *NullSigner) DetachedSign(source string, destination string) error {
+	return ioutil.WriteFile(destination, []byte{}, 0644)
+}
+
+func (n *NullSigner) ClearSign(source string, destination string) error {
+	return ioutil.WriteFile(destination, []byte{}, 0644)
+}
+
+type FakeStorageProvider struct {
+	storages map[string]aptly.PublishedStorage
+}
+
+func (p *FakeStorageProvider) GetPublishedStorage(name string) aptly.PublishedStorage {
+	storage, ok := p.storages[name]
+	if !ok {
+		panic(fmt.Sprintf("unknown storage: %#v", name))
+	}
+	return storage
+}
+
+type PublishedRepoSuite struct {
+	PackageListMixinSuite
+	repo, repo2, repo3, repo4, repo5    *PublishedRepo
+	root, root2                         string
+	provider                            *FakeStorageProvider
+	publishedStorage, publishedStorage2 *files.PublishedStorage
+	packagePool                         aptly.PackagePool
+	localRepo                           *LocalRepo
+	snapshot, snapshot2                 *Snapshot
+	db                                  database.Storage
+	factory                             *CollectionFactory
+	packageCollection                   *PackageCollection
+}
+
+var _ = Suite(&PublishedRepoSuite{})
+
+func (s *PublishedRepoSuite) SetUpTest(c *C) {
+	s.SetUpPackages()
+
+	s.db, _ = database.OpenDB(c.MkDir())
+	s.factory = NewCollectionFactory(s.db)
+
+	s.root = c.MkDir()
+	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.root2 = c.MkDir()
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{
+		"":            s.publishedStorage,
+		"files:other": s.publishedStorage2}}
+	s.packagePool = files.NewPackagePool(s.root)
+
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo.packageRefs = s.reflist
+	s.factory.RemoteRepoCollection().Add(repo)
+
+	s.localRepo = NewLocalRepo("local1", "comment1")
+	s.localRepo.packageRefs = s.reflist
+	s.factory.LocalRepoCollection().Add(s.localRepo)
+
+	s.snapshot, _ = NewSnapshotFromRepository("snap", repo)
+	s.factory.SnapshotCollection().Add(s.snapshot)
+
+	s.snapshot2, _ = NewSnapshotFromRepository("snap", repo)
+	s.factory.SnapshotCollection().Add(s.snapshot2)
+
+	s.packageCollection = s.factory.PackageCollection()
+	s.packageCollection.Update(s.p1)
+	s.packageCollection.Update(s.p2)
+	s.packageCollection.Update(s.p3)
+
+	s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+
+	s.repo2, _ = NewPublishedRepo("", "ppa", "maverick", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+
+	s.repo3, _ = NewPublishedRepo("", "linux", "natty", nil, []string{"main", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+
+	s.repo4, _ = NewPublishedRepo("", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+
+	poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
+	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
+	f, err := os.Create(poolPath)
+	c.Assert(err, IsNil)
+	f.Close()
+}
+
+func (s *PublishedRepoSuite) TearDownTest(c *C) {
+	s.db.Close()
+}
+
+func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) {
+	c.Check(s.repo.sourceItems["main"].snapshot, Equals, s.snapshot)
+	c.Check(s.repo.SourceKind, Equals, "snapshot")
+	c.Check(s.repo.Sources["main"], Equals, s.snapshot.UUID)
+	c.Check(s.repo.Components(), DeepEquals, []string{"main"})
+
+	c.Check(s.repo2.sourceItems["main"].localRepo, Equals, s.localRepo)
+	c.Check(s.repo2.SourceKind, Equals, "local")
+	c.Check(s.repo2.Sources["main"], Equals, s.localRepo.UUID)
+	c.Check(s.repo2.sourceItems["main"].packageRefs.Len(), Equals, 3)
+	c.Check(s.repo2.Components(), DeepEquals, []string{"main"})
+
+	c.Check(s.repo.RefList("main").Len(), Equals, 3)
+	c.Check(s.repo2.RefList("main").Len(), Equals, 3)
+
+	c.Check(s.repo3.Sources, DeepEquals, map[string]string{"main": s.snapshot.UUID, "contrib": s.snapshot2.UUID})
+	c.Check(s.repo3.SourceKind, Equals, "snapshot")
+	c.Check(s.repo3.sourceItems["main"].snapshot, Equals, s.snapshot)
+	c.Check(s.repo3.sourceItems["contrib"].snapshot, Equals, s.snapshot2)
+	c.Check(s.repo3.Components(), DeepEquals, []string{"contrib", "main"})
+
+	c.Check(s.repo3.RefList("main").Len(), Equals, 3)
+	c.Check(s.repo3.RefList("contrib").Len(), Equals, 3)
+
+	c.Check(func() { NewPublishedRepo("", ".", "a", nil, nil, nil, s.factory) }, PanicMatches, "publish with empty sources")
+	c.Check(func() {
+		NewPublishedRepo("", ".", "a", nil, []string{"main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	}, PanicMatches, "sources and components should be equal in size")
+	c.Check(func() {
+		NewPublishedRepo("", ".", "a", nil, []string{"main", "contrib"}, []interface{}{s.localRepo, s.snapshot2}, s.factory)
+	}, PanicMatches, "interface conversion:.*")
+
+	_, err := NewPublishedRepo("", ".", "a", nil, []string{"main", "main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	c.Check(err, ErrorMatches, "duplicate component name: main")
+}
+
+func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
+
+	for _, t := range []struct {
+		prefix        string
+		expected      string
+		errorExpected string
+	}{
+		{
+			prefix:   "ppa",
+			expected: "ppa",
+		},
+		{
+			prefix:   "",
+			expected: ".",
+		},
+		{
+			prefix:   "/",
+			expected: ".",
+		},
+		{
+			prefix:   "//",
+			expected: ".",
+		},
+		{
+			prefix:   "//ppa/",
+			expected: "ppa",
+		},
+		{
+			prefix:   "ppa/..",
+			expected: ".",
+		},
+		{
+			prefix:   "ppa/ubuntu/",
+			expected: "ppa/ubuntu",
+		},
+		{
+			prefix:   "ppa/../ubuntu/",
+			expected: "ubuntu",
+		},
+		{
+			prefix:        "../ppa/",
+			errorExpected: "invalid prefix .*",
+		},
+		{
+			prefix:        "../ppa/../ppa/",
+			errorExpected: "invalid prefix .*",
+		},
+		{
+			prefix:        "ppa/dists",
+			errorExpected: "invalid prefix .*",
+		},
+		{
+			prefix:        "ppa/pool",
+			errorExpected: "invalid prefix .*",
+		},
+	} {
+		repo, err := NewPublishedRepo("", t.prefix, "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+		if t.errorExpected != "" {
+			c.Check(err, ErrorMatches, t.errorExpected)
+		} else {
+			c.Check(repo.Prefix, Equals, t.expected)
+		}
+	}
+}
+
+func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
+	repo, err := NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "squeeze")
+	c.Check(repo.Components(), DeepEquals, []string{"main"})
+
+	repo, err = NewPublishedRepo("", "ppa", "wheezy", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "wheezy")
+	c.Check(repo.Components(), DeepEquals, []string{"main"})
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"non-free"}, []interface{}{s.snapshot}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "squeeze")
+	c.Check(repo.Components(), DeepEquals, []string{"non-free"})
+
+	repo, err = NewPublishedRepo("", "ppa", "squeeze", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "squeeze")
+	c.Check(repo.Components(), DeepEquals, []string{"main"})
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	c.Check(err, ErrorMatches, "unable to guess distribution name, please specify explicitly")
+
+	s.localRepo.DefaultDistribution = "precise"
+	s.localRepo.DefaultComponent = "contrib"
+	s.factory.LocalRepoCollection().Update(s.localRepo)
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "precise")
+	c.Check(repo.Components(), DeepEquals, []string{"contrib"})
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "squeeze")
+	c.Check(repo.Components(), DeepEquals, []string{"contrib", "main"})
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	c.Check(err, ErrorMatches, "duplicate component name: main")
+}
+
+func (s *PublishedRepoSuite) TestPublish(c *C) {
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil)
+	c.Assert(err, IsNil)
+
+	c.Check(s.repo.Architectures, DeepEquals, []string{"i386"})
+
+	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
+	c.Assert(err, IsNil)
+
+	cfr := NewControlFileReader(rf)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(st["Origin"], Equals, "ppa squeeze")
+	c.Check(st["Components"], Equals, "main")
+	c.Check(st["Architectures"], Equals, "i386")
+
+	pf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages"))
+	c.Assert(err, IsNil)
+
+	cfr = NewControlFileReader(pf)
+
+	for i := 0; i < 3; i++ {
+		st, err = cfr.ReadStanza()
+		c.Assert(err, IsNil)
+
+		c.Check(st["Filename"], Equals, "pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
+	}
+
+	st, err = cfr.ReadStanza()
+	c.Assert(err, IsNil)
+	c.Assert(st, IsNil)
+
+	drf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"))
+	c.Assert(err, IsNil)
+
+	cfr = NewControlFileReader(drf)
+	st, err = cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(st["Archive"], Equals, "squeeze")
+	c.Check(st["Architecture"], Equals, "i386")
+
+	_, err = os.Stat(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb"))
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil)
+	c.Assert(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"), PathExists)
+}
+
+func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) {
+	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil)
+	c.Assert(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/main/binary-i386/Release"), PathExists)
+}
+
+func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) {
+	err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil)
+	c.Assert(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/main/source/Release"), PathExists)
+}
+
+func (s *PublishedRepoSuite) TestPublishOtherStorage(c *C) {
+	err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil)
+	c.Assert(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), Not(PathExists))
+}
+
+func (s *PublishedRepoSuite) TestString(c *C) {
+	c.Check(s.repo.String(), Equals,
+		"ppa/squeeze [] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	c.Check(s.repo2.String(), Equals,
+		"ppa/maverick [] publishes {main: [local1]: comment1}")
+	repo, _ := NewPublishedRepo("", "", "squeeze", []string{"s390"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	c.Check(repo.String(), Equals,
+		"./squeeze [s390] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	repo, _ = NewPublishedRepo("", "", "squeeze", []string{"i386", "amd64"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	c.Check(repo.String(), Equals,
+		"./squeeze [i386, amd64] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	repo.Origin = "myorigin"
+	c.Check(repo.String(), Equals,
+		"./squeeze (origin: myorigin) [i386, amd64] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	repo.Label = "mylabel"
+	c.Check(repo.String(), Equals,
+		"./squeeze (origin: myorigin, label: mylabel) [i386, amd64] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	c.Check(s.repo3.String(), Equals,
+		"linux/natty [] publishes {contrib: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}, {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	c.Check(s.repo5.String(), Equals,
+		"files:other:ppa/maverick [source] publishes {main: [local1]: comment1}")
+}
+
+func (s *PublishedRepoSuite) TestKey(c *C) {
+	c.Check(s.repo.Key(), DeepEquals, []byte("Uppa>>squeeze"))
+	c.Check(s.repo5.Key(), DeepEquals, []byte("Ufiles:other:ppa>>maverick"))
+}
+
+func (s *PublishedRepoSuite) TestRefKey(c *C) {
+	c.Check(s.repo.RefKey(""), DeepEquals, []byte("E"+s.repo.UUID))
+	c.Check(s.repo.RefKey("main"), DeepEquals, []byte("E"+s.repo.UUID+"main"))
+}
+
+func (s *PublishedRepoSuite) TestEncodeDecode(c *C) {
+	encoded := s.repo.Encode()
+	repo := &PublishedRepo{}
+	err := repo.Decode(encoded)
+
+	s.repo.sourceItems = nil
+	c.Assert(err, IsNil)
+	c.Assert(repo, DeepEquals, s.repo)
+
+	encoded2 := s.repo2.Encode()
+	repo2 := &PublishedRepo{}
+	err = repo2.Decode(encoded2)
+
+	s.repo2.sourceItems = nil
+	c.Assert(err, IsNil)
+	c.Assert(repo2, DeepEquals, s.repo2)
+}
+
+type PublishedRepoCollectionSuite struct {
+	PackageListMixinSuite
+	db                                database.Storage
+	factory                           *CollectionFactory
+	snapshotCollection                *SnapshotCollection
+	collection                        *PublishedRepoCollection
+	snap1, snap2                      *Snapshot
+	localRepo                         *LocalRepo
+	repo1, repo2, repo3, repo4, repo5 *PublishedRepo
+}
+
+var _ = Suite(&PublishedRepoCollectionSuite{})
+
+func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
+	s.db, _ = database.OpenDB(c.MkDir())
+	s.factory = NewCollectionFactory(s.db)
+
+	s.snapshotCollection = s.factory.SnapshotCollection()
+
+	s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
+	s.snap2 = NewSnapshotFromPackageList("snap2", []*Snapshot{}, NewPackageList(), "desc2")
+
+	s.snapshotCollection.Add(s.snap1)
+	s.snapshotCollection.Add(s.snap2)
+
+	s.localRepo = NewLocalRepo("local1", "comment1")
+	s.factory.LocalRepoCollection().Add(s.localRepo)
+
+	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main", "contrib"}, []interface{}{s.snap2, s.snap1}, s.factory)
+	s.repo3, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap2}, s.factory)
+	s.repo4, _ = NewPublishedRepo("", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+
+	s.collection = s.factory.PublishedRepoCollection()
+}
+
+func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) {
+	s.db.Close()
+}
+
+func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C) {
+	r, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Assert(err, ErrorMatches, "*.not found")
+
+	c.Assert(s.collection.Add(s.repo1), IsNil)
+	c.Assert(s.collection.Add(s.repo1), ErrorMatches, ".*already exists")
+	c.Assert(s.collection.CheckDuplicate(s.repo2), IsNil)
+	c.Assert(s.collection.Add(s.repo2), IsNil)
+	c.Assert(s.collection.Add(s.repo3), ErrorMatches, ".*already exists")
+	c.Assert(s.collection.CheckDuplicate(s.repo3), Equals, s.repo1)
+	c.Assert(s.collection.Add(s.repo4), IsNil)
+	c.Assert(s.collection.Add(s.repo5), IsNil)
+
+	r, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Assert(err, IsNil)
+
+	err = s.collection.LoadComplete(r, s.factory)
+	c.Assert(err, IsNil)
+	c.Assert(r.String(), Equals, s.repo1.String())
+
+	collection := NewPublishedRepoCollection(s.db)
+	r, err = collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Assert(err, IsNil)
+
+	err = s.collection.LoadComplete(r, s.factory)
+	c.Assert(err, IsNil)
+	c.Assert(r.String(), Equals, s.repo1.String())
+
+	r, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "precise")
+	c.Check(r.String(), Equals, s.repo5.String())
+}
+
+func (s *PublishedRepoCollectionSuite) TestByUUID(c *C) {
+	r, err := s.collection.ByUUID(s.repo1.UUID)
+	c.Assert(err, ErrorMatches, "*.not found")
+
+	c.Assert(s.collection.Add(s.repo1), IsNil)
+
+	r, err = s.collection.ByUUID(s.repo1.UUID)
+	c.Assert(err, IsNil)
+
+	err = s.collection.LoadComplete(r, s.factory)
+	c.Assert(err, IsNil)
+	c.Assert(r.String(), Equals, s.repo1.String())
+}
+
+func (s *PublishedRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
+	c.Assert(s.collection.Update(s.repo1), IsNil)
+	c.Assert(s.collection.Update(s.repo4), IsNil)
+
+	collection := NewPublishedRepoCollection(s.db)
+	r, err := collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Assert(err, IsNil)
+	c.Assert(r.sourceItems["main"].snapshot, IsNil)
+	c.Assert(s.collection.LoadComplete(r, s.factory), IsNil)
+	c.Assert(r.Sources["main"], Equals, s.repo1.sourceItems["main"].snapshot.UUID)
+	c.Assert(r.RefList("main").Len(), Equals, 0)
+
+	r, err = collection.ByStoragePrefixDistribution("", "ppa", "precise")
+	c.Assert(err, IsNil)
+	c.Assert(r.sourceItems["main"].localRepo, IsNil)
+	c.Assert(s.collection.LoadComplete(r, s.factory), IsNil)
+	c.Assert(r.sourceItems["main"].localRepo.UUID, Equals, s.repo4.sourceItems["main"].localRepo.UUID)
+	c.Assert(r.sourceItems["main"].packageRefs.Len(), Equals, 0)
+	c.Assert(r.RefList("main").Len(), Equals, 0)
+}
+
+func (s *PublishedRepoCollectionSuite) TestLoadPre0_6(c *C) {
+	type oldPublishedRepo struct {
+		UUID          string
+		Prefix        string
+		Distribution  string
+		Origin        string
+		Label         string
+		Architectures []string
+		SourceKind    string
+		Component     string
+		SourceUUID    string `codec:"SnapshotUUID"`
+	}
+
+	old := oldPublishedRepo{
+		UUID:          s.repo1.UUID,
+		Prefix:        "ppa",
+		Distribution:  "anaconda",
+		Architectures: []string{"i386"},
+		SourceKind:    "local",
+		Component:     "contrib",
+		SourceUUID:    s.localRepo.UUID,
+	}
+
+	var buf bytes.Buffer
+
+	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
+	encoder.Encode(&old)
+
+	c.Assert(s.db.Put(s.repo1.Key(), buf.Bytes()), IsNil)
+	c.Assert(s.db.Put(s.repo1.RefKey(""), s.localRepo.RefList().Encode()), IsNil)
+
+	collection := NewPublishedRepoCollection(s.db)
+	repo, err := collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, IsNil)
+	c.Check(repo.Component, Equals, "")
+	c.Check(repo.SourceUUID, Equals, "")
+	c.Check(repo.Sources, DeepEquals, map[string]string{"contrib": s.localRepo.UUID})
+
+	c.Check(collection.LoadComplete(repo, s.factory), IsNil)
+	c.Check(repo.sourceItems["contrib"].localRepo.UUID, Equals, s.localRepo.UUID)
+	c.Check(repo.RefList("contrib").Len(), Equals, 0)
+}
+
+func (s *PublishedRepoCollectionSuite) TestForEachAndLen(c *C) {
+	s.collection.Add(s.repo1)
+
+	count := 0
+	err := s.collection.ForEach(func(*PublishedRepo) error {
+		count++
+		return nil
+	})
+	c.Assert(count, Equals, 1)
+	c.Assert(err, IsNil)
+
+	c.Check(s.collection.Len(), Equals, 1)
+
+	e := errors.New("c")
+
+	err = s.collection.ForEach(func(*PublishedRepo) error {
+		return e
+	})
+	c.Assert(err, Equals, e)
+}
+
+func (s *PublishedRepoCollectionSuite) TestBySnapshot(c *C) {
+	c.Check(s.collection.Add(s.repo1), IsNil)
+	c.Check(s.collection.Add(s.repo2), IsNil)
+
+	c.Check(s.collection.BySnapshot(s.snap1), DeepEquals, []*PublishedRepo{s.repo1, s.repo2})
+	c.Check(s.collection.BySnapshot(s.snap2), DeepEquals, []*PublishedRepo{s.repo2})
+}
+
+func (s *PublishedRepoCollectionSuite) TestByLocalRepo(c *C) {
+	c.Check(s.collection.Add(s.repo1), IsNil)
+	c.Check(s.collection.Add(s.repo4), IsNil)
+	c.Check(s.collection.Add(s.repo5), IsNil)
+
+	c.Check(s.collection.ByLocalRepo(s.localRepo), DeepEquals, []*PublishedRepo{s.repo4, s.repo5})
+}
+
+type PublishedRepoRemoveSuite struct {
+	PackageListMixinSuite
+	db                                  database.Storage
+	factory                             *CollectionFactory
+	snapshotCollection                  *SnapshotCollection
+	collection                          *PublishedRepoCollection
+	root, root2                         string
+	provider                            *FakeStorageProvider
+	publishedStorage, publishedStorage2 *files.PublishedStorage
+	snap1                               *Snapshot
+	repo1, repo2, repo3, repo4, repo5   *PublishedRepo
+}
+
+var _ = Suite(&PublishedRepoRemoveSuite{})
+
+func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
+	s.db, _ = database.OpenDB(c.MkDir())
+	s.factory = NewCollectionFactory(s.db)
+
+	s.snapshotCollection = s.factory.SnapshotCollection()
+
+	s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
+
+	s.snapshotCollection.Add(s.snap1)
+
+	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo3, _ = NewPublishedRepo("", "ppa", "meduza", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo4, _ = NewPublishedRepo("", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory)
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory)
+
+	s.collection = s.factory.PublishedRepoCollection()
+	s.collection.Add(s.repo1)
+	s.collection.Add(s.repo2)
+	s.collection.Add(s.repo3)
+	s.collection.Add(s.repo4)
+	s.collection.Add(s.repo5)
+
+	s.root = c.MkDir()
+	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.publishedStorage.MkDir("ppa/dists/anaconda")
+	s.publishedStorage.MkDir("ppa/dists/meduza")
+	s.publishedStorage.MkDir("ppa/dists/osminog")
+	s.publishedStorage.MkDir("ppa/pool/main")
+	s.publishedStorage.MkDir("ppa/pool/contrib")
+	s.publishedStorage.MkDir("dists/anaconda")
+	s.publishedStorage.MkDir("pool/main")
+
+	s.root2 = c.MkDir()
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.publishedStorage2.MkDir("ppa/dists/osminog")
+	s.publishedStorage2.MkDir("ppa/pool/contrib")
+
+	s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{
+		"":            s.publishedStorage,
+		"files:other": s.publishedStorage2}}
+}
+
+func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
+	s.db.Close()
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
+	s.repo1.RemoveFiles(s.provider, false, []string{}, nil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
+	s.repo1.RemoveFiles(s.provider, false, []string{"main"}, nil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithTwoPools(c *C) {
+	s.repo1.RemoveFiles(s.provider, false, []string{"main", "contrib"}, nil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
+	s.repo1.RemoveFiles(s.provider, true, []string{"main"}, nil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
+	s.repo2.RemoveFiles(s.provider, true, []string{"main"}, nil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	collection := NewPublishedRepoCollection(s.db)
+	_, err = collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
+	c.Check(err, ErrorMatches, ".*not found")
+
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil)
+	c.Check(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
+	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	collection := NewPublishedRepoCollection(s.db)
+	_, err = collection.ByStoragePrefixDistribution("", ".", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) {
+	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	collection := NewPublishedRepoCollection(s.db)
+	_, err = collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
+}

deb/query.go

@@ -0,0 +1,262 @@
+package deb
+
+import (
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// PackageQuery is interface of predicate on Package
+type PackageQuery interface {
+	// Matches calculates match of condition against package
+	Matches(pkg *Package) bool
+	// Fast returns if search strategy is possible for this query
+	Fast() bool
+	// Query performs search on package list
+	Query(list *PackageList) *PackageList
+	// String interface
+	String() string
+}
+
+// OrQuery is L | R
+type OrQuery struct {
+	L, R PackageQuery
+}
+
+// AndQuery is L , R
+type AndQuery struct {
+	L, R PackageQuery
+}
+
+// NotQuery is ! Q
+type NotQuery struct {
+	Q PackageQuery
+}
+
+// FieldQuery is generic request against field
+type FieldQuery struct {
+	Field    string
+	Relation int
+	Value    string
+	Regexp   *regexp.Regexp `codec:"-"`
+}
+
+// PkgQuery is search request against specific package
+type PkgQuery struct {
+	Pkg     string
+	Version string
+	Arch    string
+}
+
+// DependencyQuery is generic Debian-dependency like query
+type DependencyQuery struct {
+	Dep Dependency
+}
+
+// Matches if any of L, R matches
+func (q *OrQuery) Matches(pkg *Package) bool {
+	return q.L.Matches(pkg) || q.R.Matches(pkg)
+}
+
+// Fast is true only if both parts are fast
+func (q *OrQuery) Fast() bool {
+	return q.L.Fast() && q.R.Fast()
+}
+
+// Query strategy depends on nodes
+func (q *OrQuery) Query(list *PackageList) (result *PackageList) {
+	if q.Fast() {
+		result = q.L.Query(list)
+		result.Append(q.R.Query(list))
+	} else {
+		result = list.Scan(q)
+	}
+	return
+}
+
+// String interface
+func (q *OrQuery) String() string {
+	return fmt.Sprintf("(%s) | (%s)", q.L, q.R)
+}
+
+// Matches if both of L, R matches
+func (q *AndQuery) Matches(pkg *Package) bool {
+	return q.L.Matches(pkg) && q.R.Matches(pkg)
+}
+
+// Fast is true if any of the parts are fast
+func (q *AndQuery) Fast() bool {
+	return q.L.Fast() || q.R.Fast()
+}
+
+// Query strategy depends on nodes
+func (q *AndQuery) Query(list *PackageList) (result *PackageList) {
+	if !q.Fast() {
+		result = list.Scan(q)
+	} else {
+		if q.L.Fast() {
+			result = q.L.Query(list)
+			result = result.Scan(q.R)
+		} else {
+			result = q.R.Query(list)
+			result = result.Scan(q.L)
+		}
+	}
+	return
+}
+
+// String interface
+func (q *AndQuery) String() string {
+	return fmt.Sprintf("(%s), (%s)", q.L, q.R)
+}
+
+// Matches if not matches
+func (q *NotQuery) Matches(pkg *Package) bool {
+	return !q.Q.Matches(pkg)
+}
+
+// Fast is false
+func (q *NotQuery) Fast() bool {
+	return false
+}
+
+// Query strategy is scan always
+func (q *NotQuery) Query(list *PackageList) (result *PackageList) {
+	result = list.Scan(q)
+	return
+}
+
+// String interface
+func (q *NotQuery) String() string {
+	return fmt.Sprintf("!(%s)", q.Q)
+}
+
+// Matches on generic field
+func (q *FieldQuery) Matches(pkg *Package) bool {
+	if q.Field == "$Version" {
+		return pkg.MatchesDependency(Dependency{Pkg: pkg.Name, Relation: q.Relation, Version: q.Value, Regexp: q.Regexp})
+	}
+	if q.Field == "$Architecture" && q.Relation == VersionEqual {
+		return pkg.MatchesArchitecture(q.Value)
+	}
+
+	field := pkg.GetField(q.Field)
+
+	switch q.Relation {
+	case VersionDontCare:
+		return field != ""
+	case VersionEqual:
+		return field == q.Value
+	case VersionGreater:
+		return field > q.Value
+	case VersionGreaterOrEqual:
+		return field >= q.Value
+	case VersionLess:
+		return field < q.Value
+	case VersionLessOrEqual:
+		return field <= q.Value
+	case VersionPatternMatch:
+		matched, err := filepath.Match(q.Value, field)
+		return err == nil && matched
+	case VersionRegexp:
+		if q.Regexp == nil {
+			q.Regexp = regexp.MustCompile(q.Value)
+		}
+		return q.Regexp.FindStringIndex(field) != nil
+
+	}
+	panic("unknown relation")
+}
+
+// Query runs iteration through list
+func (q *FieldQuery) Query(list *PackageList) (result *PackageList) {
+	result = list.Scan(q)
+	return
+}
+
+// Fast depends on the query
+func (q *FieldQuery) Fast() bool {
+	return false
+}
+
+// String interface
+func (q *FieldQuery) String() string {
+	escape := func(val string) string {
+		if strings.IndexAny(val, "()|,!{} \t\n") != -1 {
+			return "'" + strings.Replace(strings.Replace(val, "\\", "\\\\", -1), "'", "\\'", -1) + "'"
+		}
+		return val
+	}
+
+	var op string
+	switch q.Relation {
+	case VersionEqual:
+		op = "="
+	case VersionGreater:
+		op = ">>"
+	case VersionLess:
+		op = "<<"
+	case VersionRegexp:
+		op = "~"
+	case VersionPatternMatch:
+		op = "%"
+	case VersionGreaterOrEqual:
+		op = ">="
+	case VersionLessOrEqual:
+		op = "<="
+	}
+	return fmt.Sprintf("%s (%s %s)", escape(q.Field), op, escape(q.Value))
+}
+
+// Matches on dependency condition
+func (q *DependencyQuery) Matches(pkg *Package) bool {
+	return pkg.MatchesDependency(q.Dep)
+}
+
+// Fast is always true for dependency query
+func (q *DependencyQuery) Fast() bool {
+	return true
+}
+
+// Query runs PackageList.Search
+func (q *DependencyQuery) Query(list *PackageList) (result *PackageList) {
+	result = NewPackageList()
+	for _, pkg := range list.Search(q.Dep, true) {
+		result.Add(pkg)
+	}
+
+	return
+}
+
+// String interface
+func (q *DependencyQuery) String() string {
+	return q.Dep.String()
+}
+
+// Matches on specific properties
+func (q *PkgQuery) Matches(pkg *Package) bool {
+	return pkg.Name == q.Pkg && pkg.Version == q.Version && pkg.Architecture == q.Arch
+}
+
+// Fast is always true for package query
+func (q *PkgQuery) Fast() bool {
+	return true
+}
+
+// Query looks up specific package
+func (q *PkgQuery) Query(list *PackageList) (result *PackageList) {
+	result = NewPackageList()
+
+	pkg := list.packages["P"+q.Arch+" "+q.Pkg+" "+q.Version]
+	if pkg != nil {
+		result.Add(pkg)
+	}
+
+	return
+}
+
+// String interface
+func (q *PkgQuery) String() string {
+	return fmt.Sprintf("%s_%s_%s", q.Pkg, q.Version, q.Arch)
+}

deb/reflist.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -194,34 +194,40 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle
 				}
 			}
 
-			// is pl & pr the same package, but different version?
-			if pl.Name == pr.Name && pl.Architecture == pr.Architecture {
-				result = append(result, PackageDiff{Left: pl, Right: pr})
-				il++
-				ir++
-				pl, pr = nil, nil
-			} else {
-				// otherwise pl or pr is missing on one of the sides
-				if rel < 0 {
+			// otherwise pl or pr is missing on one of the sides
+			if rel < 0 {
+				// compaction: +(,A) -(B,) --> !(A,B)
+				if len(result) > 0 && result[len(result)-1].Left == nil && result[len(result)-1].Right.Name == pl.Name &&
+					result[len(result)-1].Right.Architecture == pl.Architecture {
+					result[len(result)-1] = PackageDiff{Left: pl, Right: result[len(result)-1].Right}
+				} else {
 					result = append(result, PackageDiff{Left: pl, Right: nil})
-					il++
-					pl = nil
+				}
+				il++
+				pl = nil
+			} else {
+				// compaction: -(A,) +(,B) --> !(A,B)
+				if len(result) > 0 && result[len(result)-1].Right == nil && result[len(result)-1].Left.Name == pr.Name &&
+					result[len(result)-1].Left.Architecture == pr.Architecture {
+					result[len(result)-1] = PackageDiff{Left: result[len(result)-1].Left, Right: pr}
 				} else {
 					result = append(result, PackageDiff{Left: nil, Right: pr})
-					ir++
-					pr = nil
 				}
+				ir++
+				pr = nil
 			}
-
 		}
 	}
 
 	return
 }
 
-// Merge merges reflist r into current reflist. If overrideMatching, merge replaces matching packages (by architecture/name)
-// with reference from r, otherwise all packages are saved.
+// Merge merges reflist r into current reflist. If overrideMatching, merge
+// replaces matching packages (by architecture/name) with reference from r.
+// Otherwise, all packages are saved.
 func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result *PackageRefList) {
+	var overriddenArch, overridenName []byte
+
 	// pointer to left and right reflists
 	il, ir := 0, 0
 	// length of reflists
@@ -253,6 +259,8 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result
 			result.Refs = append(result.Refs, l.Refs[il])
 			il++
 			ir++
+			overridenName = nil
+			overriddenArch = nil
 		} else {
 			if overrideMatching {
 				partsL := bytes.Split(rl, []byte(" "))
@@ -261,11 +269,19 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result
 				partsR := bytes.Split(rr, []byte(" "))
 				archR, nameR := partsR[0][1:], partsR[1]
 
-				if bytes.Compare(archL, archR) == 0 && bytes.Compare(nameL, nameR) == 0 {
+				if bytes.Equal(archL, overriddenArch) && bytes.Equal(nameL, overridenName) {
+					// this package has already been overriden on the right
+					il++
+					continue
+				}
+
+				if bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) {
 					// override with package from the right
 					result.Refs = append(result.Refs, r.Refs[ir])
 					il++
 					ir++
+					overriddenArch = archL
+					overridenName = nameL
 					continue
 				}
 			}
@@ -277,10 +293,50 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result
 			} else {
 				result.Refs = append(result.Refs, r.Refs[ir])
 				ir++
+				overridenName = nil
+				overriddenArch = nil
 			}
-
 		}
 	}
 
 	return
 }
+
+// FilterLatestRefs takes in a reflist with potentially multiples of the same
+// packages and reduces it to only the latest of each package. The operations
+// are done in-place. This implements a "latest wins" approach which can be used
+// while merging two or more snapshots together.
+func FilterLatestRefs(r *PackageRefList) {
+	var (
+		lastArch, lastName, lastVer []byte
+		arch, name, ver             []byte
+		parts                       [][]byte
+	)
+
+	for i := 0; i < len(r.Refs); i++ {
+		parts = bytes.Split(r.Refs[i][1:], []byte(" "))
+		arch, name, ver = parts[0], parts[1], parts[2]
+
+		if bytes.Equal(arch, lastArch) && bytes.Equal(name, lastName) {
+			// Two packages are identical, check version and only one wins
+			vres := CompareVersions(string(ver), string(lastVer))
+
+			// Remove the older refs from the result
+			if vres > 0 {
+				// ver[i] > ver[i-1], remove element i-1
+				r.Refs = append(r.Refs[:i-1], r.Refs[i:]...)
+			} else {
+				// ver[i] < ver[i-1], remove element i
+				r.Refs = append(r.Refs[:i], r.Refs[i+1:]...)
+				arch, name, ver = lastArch, lastName, lastVer
+			}
+
+			// Compensate for the reduced set
+			i -= 1
+		}
+
+		lastArch, lastName, lastVer = arch, name, ver
+	}
+
+	return
+}

deb/reflist_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"errors"
@@ -14,6 +14,14 @@ type PackageRefListSuite struct {
 
 var _ = Suite(&PackageRefListSuite{})
 
+func toStrSlice(reflist *PackageRefList) (result []string) {
+	result = make([]string, reflist.Len())
+	for i, r := range reflist.Refs {
+		result[i] = string(r)
+	}
+	return
+}
+
 func (s *PackageRefListSuite) SetUpTest(c *C) {
 	s.list = NewPackageList()
 
@@ -23,7 +31,7 @@ func (s *PackageRefListSuite) SetUpTest(c *C) {
 	stanza["Package"] = "mars-invaders"
 	s.p3 = NewPackageFromControlFile(stanza)
 	stanza = packageStanza.Copy()
-	stanza["Size"] = "42"
+	stanza["Source"] = "unknown-planet"
 	s.p4 = NewPackageFromControlFile(stanza)
 	stanza = packageStanza.Copy()
 	stanza["Package"] = "lonely-strangers"
@@ -250,14 +258,6 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	reflistA := NewPackageRefListFromPackageList(listA)
 	reflistB := NewPackageRefListFromPackageList(listB)
 
-	toStrSlice := func(reflist *PackageRefList) (result []string) {
-		result = make([]string, reflist.Len())
-		for i, r := range reflist.Refs {
-			result[i] = string(r)
-		}
-		return
-	}
-
 	mergeAB := reflistA.Merge(reflistB, true)
 	mergeBA := reflistB.Merge(reflistA, true)
 
@@ -273,3 +273,32 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	c.Check(toStrSlice(mergeBAall), DeepEquals,
 		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"})
 }
+
+func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) {
+	packages := []*Package{
+		&Package{Name: "lib", Version: "1.0", Architecture: "i386"},
+		&Package{Name: "lib", Version: "1.2~bp1", Architecture: "i386"},
+		&Package{Name: "lib", Version: "1.2", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.2", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.3", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.3~bp2", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.5", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.6", Architecture: "i386"},
+	}
+
+	rl := NewPackageList()
+	rl.Add(packages[0])
+	rl.Add(packages[1])
+	rl.Add(packages[2])
+	rl.Add(packages[3])
+	rl.Add(packages[4])
+	rl.Add(packages[5])
+	rl.Add(packages[6])
+	rl.Add(packages[7])
+
+	result := NewPackageRefListFromPackageList(rl)
+	FilterLatestRefs(result)
+
+	c.Check(toStrSlice(result), DeepEquals,
+		[]string{"Pi386 dpkg 1.6", "Pi386 lib 1.2"})
+}

deb/remote.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -13,6 +13,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -42,6 +43,10 @@ type RemoteRepo struct {
 	LastDownloadDate time.Time
 	// Checksums for release files
 	ReleaseFiles map[string]utils.ChecksumInfo
+	// Filter for packages
+	Filter string
+	// FilterWithDeps to include dependencies from filter query
+	FilterWithDeps bool
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 	// Parsed archived root
@@ -66,9 +71,11 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone
 		return nil, err
 	}
 
-	if result.Distribution == "." || result.Distribution == "./" {
+	if strings.HasSuffix(result.Distribution, "/") || strings.HasPrefix(result.Distribution, ".") {
 		// flat repo
-		result.Distribution = ""
+		if !strings.HasPrefix(result.Distribution, ".") {
+			result.Distribution = "./" + result.Distribution
+		}
 		result.Architectures = nil
 		if len(result.Components) > 0 {
 			return nil, fmt.Errorf("components aren't supported for flat repos")
@@ -106,7 +113,9 @@ func (repo *RemoteRepo) String() string {
 
 // IsFlat determines if repository is flat
 func (repo *RemoteRepo) IsFlat() bool {
-	return repo.Distribution == ""
+	// aptly < 0.5.1 had Distribution = "" for flat repos
+	// aptly >= 0.5.1 had Distribution = "./[path]/" for flat repos
+	return repo.Distribution == "" || (strings.HasPrefix(repo.Distribution, ".") && strings.HasSuffix(repo.Distribution, "/"))
 }
 
 // NumPackages return number of packages retrived from remote repo
@@ -129,7 +138,7 @@ func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
 	if !repo.IsFlat() {
 		path = &url.URL{Path: fmt.Sprintf("dists/%s/%s", repo.Distribution, name)}
 	} else {
-		path = &url.URL{Path: name}
+		path = &url.URL{Path: filepath.Join(repo.Distribution, name)}
 	}
 
 	return repo.archiveRootURL.ResolveReference(path)
@@ -137,13 +146,13 @@ func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
 
 // FlatBinaryURL returns URL to Packages files for flat repo
 func (repo *RemoteRepo) FlatBinaryURL() *url.URL {
-	path := &url.URL{Path: "Packages"}
+	path := &url.URL{Path: filepath.Join(repo.Distribution, "Packages")}
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
 // FlatSourcesURL returns URL to Sources files for flat repo
 func (repo *RemoteRepo) FlatSourcesURL() *url.URL {
-	path := &url.URL{Path: "Sources"}
+	path := &url.URL{Path: filepath.Join(repo.Distribution, "Sources")}
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
@@ -170,8 +179,8 @@ func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
 // Fetch updates information about repository
 func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error {
 	var (
-		release *os.File
-		err     error
+		release, inrelease, releasesig *os.File
+		err                            error
 	)
 
 	if verifier == nil {
@@ -182,7 +191,7 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error
 		}
 	} else {
 		// 1. try InRelease file
-		inrelease, err := http.DownloadTemp(d, repo.ReleaseURL("InRelease").String())
+		inrelease, err = http.DownloadTemp(d, repo.ReleaseURL("InRelease").String())
 		if err != nil {
 			goto splitsignature
 		}
@@ -209,7 +218,7 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error
 			return err
 		}
 
-		releasesig, err := http.DownloadTemp(d, repo.ReleaseURL("Release.gpg").String())
+		releasesig, err = http.DownloadTemp(d, repo.ReleaseURL("Release.gpg").String())
 		if err != nil {
 			return err
 		}
@@ -275,7 +284,8 @@ ok:
 				return fmt.Errorf("unparseable hash sum line: %#v", line)
 			}
 
-			size, err := strconv.ParseInt(parts[1], 10, 64)
+			var size int64
+			size, err = strconv.ParseInt(parts[1], 10, 64)
 			if err != nil {
 				return fmt.Errorf("unable to parse size: %s", err)
 			}
@@ -314,7 +324,8 @@ ok:
 }
 
 // Download downloads all repo files
-func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, packageCollection *PackageCollection, packagePool aptly.PackagePool, ignoreMismatch bool) error {
+func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
+	packagePool aptly.PackagePool, ignoreMismatch bool, dependencyOptions int, filterQuery PackageQuery) error {
 	list := NewPackageList()
 
 	progress.Printf("Downloading & parsing package files...\n")
@@ -378,7 +389,7 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, pa
 				return err
 			}
 
-			err = packageCollection.Update(p)
+			err = collectionFactory.PackageCollection().Update(p)
 			if err != nil {
 				return err
 			}
@@ -387,6 +398,25 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, pa
 		progress.ShutdownBar()
 	}
 
+	var err error
+
+	if repo.Filter != "" {
+		progress.Printf("Applying filter...\n")
+
+		list.PrepareIndex()
+
+		emptyList := NewPackageList()
+		emptyList.PrepareIndex()
+
+		origPackages := list.Len()
+		list, err = list.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures)
+		if err != nil {
+			return err
+		}
+
+		progress.Printf("Packages filtered: %d -> %d.\n", origPackages, list.Len())
+	}
+
 	progress.Printf("Building download queue...\n")
 
 	// Build download queue
@@ -394,10 +424,10 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, pa
 	count := 0
 	downloadSize := int64(0)
 
-	err := list.ForEach(func(p *Package) error {
-		list, err := p.DownloadList(packagePool)
-		if err != nil {
-			return err
+	err = list.ForEach(func(p *Package) error {
+		list, err2 := p.DownloadList(packagePool)
+		if err2 != nil {
+			return err2
 		}
 		p.files = nil
 
@@ -421,7 +451,7 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, pa
 	// free up package list, we don't need it after this point
 	list = nil
 
-	progress.Printf("Download queue: %d items, %.2f GiB size\n", count, float64(downloadSize)/(1024.0*1024.0*1024.0))
+	progress.Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
 
 	progress.InitBar(downloadSize, true)
 

deb/remote_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"errors"
@@ -72,7 +72,7 @@ type RemoteRepoSuite struct {
 	downloader        *http.FakeDownloader
 	progress          aptly.Progress
 	db                database.Storage
-	packageCollection *PackageCollection
+	collectionFactory *CollectionFactory
 	packagePool       aptly.PackagePool
 }
 
@@ -84,7 +84,7 @@ func (s *RemoteRepoSuite) SetUpTest(c *C) {
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress()
 	s.db, _ = database.OpenDB(c.MkDir())
-	s.packageCollection = NewPackageCollection(s.db)
+	s.collectionFactory = NewCollectionFactory(s.db)
 	s.packagePool = files.NewPackagePool(c.MkDir())
 	s.SetUpPackages()
 	s.progress.Start()
@@ -101,10 +101,15 @@ func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestFlatCreation(c *C) {
-	c.Check(s.flat.Distribution, Equals, "")
+	c.Check(s.flat.IsFlat(), Equals, true)
+	c.Check(s.flat.Distribution, Equals, "./")
 	c.Check(s.flat.Architectures, IsNil)
 	c.Check(s.flat.Components, IsNil)
 
+	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false)
+	c.Check(flat2.IsFlat(), Equals, true)
+	c.Check(flat2.Distribution, Equals, "./binary/")
+
 	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false)
 	c.Check(err, ErrorMatches, "components aren't supported for flat repos")
 }
@@ -246,12 +251,12 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
 
-	err = s.repo.Download(s.progress, s.downloader, s.packageCollection, s.packagePool, false)
+	err = s.repo.Download(s.progress, s.downloader, s.collectionFactory, s.packagePool, false, 0, nil)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 	c.Assert(s.repo.packageRefs, NotNil)
 
-	pkg, err := s.packageCollection.ByKey(s.repo.packageRefs.Refs[0])
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	result, err := pkg.VerifyFiles(s.packagePool)
@@ -279,12 +284,12 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz", "abcd")
 	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz", "abcde")
 
-	err = s.repo.Download(s.progress, s.downloader, s.packageCollection, s.packagePool, false)
+	err = s.repo.Download(s.progress, s.downloader, s.collectionFactory, s.packagePool, false, 0, nil)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 	c.Assert(s.repo.packageRefs, NotNil)
 
-	pkg, err := s.packageCollection.ByKey(s.repo.packageRefs.Refs[0])
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	result, err := pkg.VerifyFiles(s.packagePool)
@@ -293,7 +298,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 
 	c.Check(pkg.Name, Equals, "amanda-client")
 
-	pkg, err = s.packageCollection.ByKey(s.repo.packageRefs.Refs[1])
+	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
 
 	result, err = pkg.VerifyFiles(s.packagePool)
@@ -314,12 +319,12 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.Download(s.progress, downloader, s.packageCollection, s.packagePool, false)
+	err = s.flat.Download(s.progress, downloader, s.collectionFactory, s.packagePool, false, 0, nil)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 	c.Assert(s.flat.packageRefs, NotNil)
 
-	pkg, err := s.packageCollection.ByKey(s.flat.packageRefs.Refs[0])
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	result, err := pkg.VerifyFiles(s.packagePool)
@@ -348,12 +353,12 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.Download(s.progress, downloader, s.packageCollection, s.packagePool, false)
+	err = s.flat.Download(s.progress, downloader, s.collectionFactory, s.packagePool, false, 0, nil)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 	c.Assert(s.flat.packageRefs, NotNil)
 
-	pkg, err := s.packageCollection.ByKey(s.flat.packageRefs.Refs[0])
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	result, err := pkg.VerifyFiles(s.packagePool)
@@ -362,7 +367,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 
 	c.Check(pkg.Name, Equals, "amanda-client")
 
-	pkg, err = s.packageCollection.ByKey(s.flat.packageRefs.Refs[1])
+	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
 
 	result, err = pkg.VerifyFiles(s.packagePool)

deb/snapshot.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -178,7 +178,10 @@ func (collection *SnapshotCollection) Update(snapshot *Snapshot) error {
 	if err != nil {
 		return err
 	}
-	return collection.db.Put(snapshot.RefKey(), snapshot.packageRefs.Encode())
+	if snapshot.packageRefs != nil {
+		return collection.db.Put(snapshot.RefKey(), snapshot.packageRefs.Encode())
+	}
+	return nil
 }
 
 // LoadComplete loads additional information about snapshot

deb/snapshot_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"errors"

deb/version.go

@@ -1,7 +1,8 @@
-package debian
+package deb
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 	"unicode"
@@ -178,6 +179,8 @@ const (
 	VersionEqual
 	VersionGreaterOrEqual
 	VersionGreater
+	VersionPatternMatch
+	VersionRegexp
 )
 
 // Dependency is a parsed version of Debian dependency to package
@@ -186,6 +189,7 @@ type Dependency struct {
 	Relation     int
 	Version      string
 	Architecture string
+	Regexp       *regexp.Regexp
 }
 
 // Hash calculates some predefined unique ID of Dependency
@@ -207,6 +211,10 @@ func (d *Dependency) String() string {
 		rel = ">="
 	case VersionLessOrEqual:
 		rel = "<="
+	case VersionPatternMatch:
+		rel = "%"
+	case VersionRegexp:
+		rel = "~"
 	case VersionDontCare:
 		return fmt.Sprintf("%s [%s]", d.Pkg, d.Architecture)
 	}

deb/version_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	. "launchpad.net/gocheck"

debian/debian.go

@@ -1,2 +0,0 @@
-// Package debian implements Debian-specific repository handling
-package debian

debian/publish.go

@@ -1,467 +0,0 @@
-package debian
-
-import (
-	"bufio"
-	"bytes"
-	"code.google.com/p/go-uuid/uuid"
-	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/utils"
-	"github.com/ugorji/go/codec"
-	"log"
-	"path/filepath"
-	"sort"
-	"strings"
-	"time"
-)
-
-// PublishedRepo is a published for http/ftp representation of snapshot as Debian repository
-type PublishedRepo struct {
-	// Internal unique ID
-	UUID string
-	// Prefix & distribution should be unique across all published repositories
-	Prefix       string
-	Distribution string
-	Component    string
-	// Architectures is a list of all architectures published
-	Architectures []string
-	// Snapshot as a source of publishing
-	SnapshotUUID string
-
-	snapshot *Snapshot
-}
-
-// NewPublishedRepo creates new published repository
-func NewPublishedRepo(prefix string, distribution string, component string, architectures []string, snapshot *Snapshot) (*PublishedRepo, error) {
-	prefix = filepath.Clean(prefix)
-	if strings.HasPrefix(prefix, "/") {
-		prefix = prefix[1:]
-	}
-	if strings.HasSuffix(prefix, "/") {
-		prefix = prefix[:len(prefix)-1]
-	}
-	prefix = filepath.Clean(prefix)
-
-	for _, component := range strings.Split(prefix, "/") {
-		if component == ".." || component == "dists" || component == "pool" {
-			return nil, fmt.Errorf("invalid prefix %s", prefix)
-		}
-	}
-
-	return &PublishedRepo{
-		UUID:          uuid.New(),
-		Prefix:        prefix,
-		Distribution:  distribution,
-		Component:     component,
-		Architectures: architectures,
-		SnapshotUUID:  snapshot.UUID,
-		snapshot:      snapshot,
-	}, nil
-}
-
-// String returns human-readable represenation of PublishedRepo
-func (p *PublishedRepo) String() string {
-	var archs string
-
-	if len(p.Architectures) > 0 {
-		archs = fmt.Sprintf(" [%s]", strings.Join(p.Architectures, ", "))
-	}
-
-	return fmt.Sprintf("%s/%s (%s)%s publishes %s", p.Prefix, p.Distribution, p.Component, archs, p.snapshot.String())
-}
-
-// Key returns unique key identifying PublishedRepo
-func (p *PublishedRepo) Key() []byte {
-	return []byte("U" + p.Prefix + ">>" + p.Distribution)
-}
-
-// Encode does msgpack encoding of PublishedRepo
-func (p *PublishedRepo) Encode() []byte {
-	var buf bytes.Buffer
-
-	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
-	encoder.Encode(p)
-
-	return buf.Bytes()
-}
-
-// Decode decodes msgpack representation into PublishedRepo
-func (p *PublishedRepo) Decode(input []byte) error {
-	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
-	return decoder.Decode(p)
-}
-
-// Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
-func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage aptly.PublishedStorage, packageCollection *PackageCollection, signer utils.Signer, progress aptly.Progress) error {
-	err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
-	if err != nil {
-		return err
-	}
-	basePath := filepath.Join(p.Prefix, "dists", p.Distribution)
-	err = publishedStorage.MkDir(basePath)
-	if err != nil {
-		return err
-	}
-
-	if progress != nil {
-		progress.Printf("Loading packages...\n")
-	}
-
-	// Load all packages
-	list, err := NewPackageListFromRefList(p.snapshot.RefList(), packageCollection, progress)
-	if err != nil {
-		return fmt.Errorf("unable to load packages: %s", err)
-	}
-
-	if list.Len() == 0 {
-		return fmt.Errorf("snapshot is empty")
-	}
-
-	if len(p.Architectures) == 0 {
-		p.Architectures = list.Architectures(true)
-	}
-
-	if len(p.Architectures) == 0 {
-		return fmt.Errorf("unable to figure out list of architectures, please supply explicit list")
-	}
-
-	sort.Strings(p.Architectures)
-
-	generatedFiles := map[string]utils.ChecksumInfo{}
-
-	if progress != nil {
-		progress.Printf("Generating metadata files and linking package files...\n")
-	}
-
-	// For all architectures, generate release file
-	for _, arch := range p.Architectures {
-		if progress != nil {
-			progress.InitBar(int64(list.Len()), false)
-		}
-
-		var relativePath string
-		if arch == "source" {
-			relativePath = filepath.Join(p.Component, "source", "Sources")
-		} else {
-			relativePath = filepath.Join(p.Component, fmt.Sprintf("binary-%s", arch), "Packages")
-		}
-		err = publishedStorage.MkDir(filepath.Dir(filepath.Join(basePath, relativePath)))
-		if err != nil {
-			return err
-		}
-
-		packagesFile, err := publishedStorage.CreateFile(filepath.Join(basePath, relativePath))
-		if err != nil {
-			return fmt.Errorf("unable to creates Packages file: %s", err)
-		}
-
-		bufWriter := bufio.NewWriter(packagesFile)
-
-		err = list.ForEach(func(pkg *Package) error {
-			if progress != nil {
-				progress.AddBar(1)
-			}
-			if pkg.MatchesArchitecture(arch) {
-				err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, p.Component)
-				if err != nil {
-					return err
-				}
-
-				err = pkg.Stanza().WriteTo(bufWriter)
-				if err != nil {
-					return err
-				}
-				err = bufWriter.WriteByte('\n')
-				if err != nil {
-					return err
-				}
-
-				pkg.files = nil
-				pkg.deps = nil
-				pkg.extra = nil
-
-			}
-
-			return nil
-		})
-
-		if err != nil {
-			return fmt.Errorf("unable to process packages: %s", err)
-		}
-
-		err = bufWriter.Flush()
-		if err != nil {
-			return fmt.Errorf("unable to write Packages file: %s", err)
-		}
-
-		err = utils.CompressFile(packagesFile)
-		if err != nil {
-			return fmt.Errorf("unable to compress Packages files: %s", err)
-		}
-
-		packagesFile.Close()
-
-		checksumInfo, err := publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath))
-		if err != nil {
-			return fmt.Errorf("unable to collect checksums: %s", err)
-		}
-		generatedFiles[relativePath] = checksumInfo
-
-		checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+".gz"))
-		if err != nil {
-			return fmt.Errorf("unable to collect checksums: %s", err)
-		}
-		generatedFiles[relativePath+".gz"] = checksumInfo
-
-		checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+".bz2"))
-		if err != nil {
-			return fmt.Errorf("unable to collect checksums: %s", err)
-		}
-		generatedFiles[relativePath+".bz2"] = checksumInfo
-
-		if progress != nil {
-			progress.ShutdownBar()
-		}
-	}
-
-	release := make(Stanza)
-	release["Origin"] = p.Prefix + " " + p.Distribution
-	release["Label"] = p.Prefix + " " + p.Distribution
-	release["Codename"] = p.Distribution
-	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
-	release["Components"] = p.Component
-	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{"source"}), " ")
-	release["Description"] = " Generated by aptly\n"
-	release["MD5Sum"] = "\n"
-	release["SHA1"] = "\n"
-	release["SHA256"] = "\n"
-
-	for path, info := range generatedFiles {
-		release["MD5Sum"] += fmt.Sprintf(" %s %8d %s\n", info.MD5, info.Size, path)
-		release["SHA1"] += fmt.Sprintf(" %s %8d %s\n", info.SHA1, info.Size, path)
-		release["SHA256"] += fmt.Sprintf(" %s %8d %s\n", info.SHA256, info.Size, path)
-	}
-
-	releaseFile, err := publishedStorage.CreateFile(filepath.Join(basePath, "Release"))
-	if err != nil {
-		return fmt.Errorf("unable to create Release file: %s", err)
-	}
-
-	bufWriter := bufio.NewWriter(releaseFile)
-
-	err = release.WriteTo(bufWriter)
-	if err != nil {
-		return fmt.Errorf("unable to create Release file: %s", err)
-	}
-
-	err = bufWriter.Flush()
-	if err != nil {
-		return fmt.Errorf("unable to create Release file: %s", err)
-	}
-
-	releaseFilename := releaseFile.Name()
-	releaseFile.Close()
-
-	// Signing files might output to console, so flush progress writer first
-	if progress != nil {
-		progress.Flush()
-	}
-
-	if signer != nil {
-		err = signer.DetachedSign(releaseFilename, releaseFilename+".gpg")
-		if err != nil {
-			return fmt.Errorf("unable to sign Release file: %s", err)
-		}
-
-		err = signer.ClearSign(releaseFilename, filepath.Join(filepath.Dir(releaseFilename), "InRelease"))
-		if err != nil {
-			return fmt.Errorf("unable to sign Release file: %s", err)
-		}
-	}
-
-	return nil
-}
-
-// RemoveFiles removes files that were created by Publish
-//
-// It can remove prefix fully, and part of pool (for specific component)
-func (p *PublishedRepo) RemoveFiles(publishedStorage aptly.PublishedStorage, removePrefix, removePoolComponent bool) error {
-	if removePrefix {
-		err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists"))
-		if err != nil {
-			return err
-		}
-
-		return publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool"))
-	}
-
-	err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution))
-	if err != nil {
-		return err
-	}
-
-	if removePoolComponent {
-		err = publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool", p.Component))
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// PublishedRepoCollection does listing, updating/adding/deleting of PublishedRepos
-type PublishedRepoCollection struct {
-	db   database.Storage
-	list []*PublishedRepo
-}
-
-// NewPublishedRepoCollection loads PublishedRepos from DB and makes up collection
-func NewPublishedRepoCollection(db database.Storage) *PublishedRepoCollection {
-	result := &PublishedRepoCollection{
-		db: db,
-	}
-
-	blobs := db.FetchByPrefix([]byte("U"))
-	result.list = make([]*PublishedRepo, 0, len(blobs))
-
-	for _, blob := range blobs {
-		r := &PublishedRepo{}
-		if err := r.Decode(blob); err != nil {
-			log.Printf("Error decoding published repo: %s\n", err)
-		} else {
-			result.list = append(result.list, r)
-		}
-	}
-
-	return result
-}
-
-// Add appends new repo to collection and saves it
-func (collection *PublishedRepoCollection) Add(repo *PublishedRepo) error {
-	if collection.CheckDuplicate(repo) != nil {
-		return fmt.Errorf("published repo with prefix/distribution %s/%s already exists", repo.Prefix, repo.Distribution)
-	}
-
-	err := collection.Update(repo)
-	if err != nil {
-		return err
-	}
-
-	collection.list = append(collection.list, repo)
-	return nil
-}
-
-// CheckDuplicate verifies that there's no published repo with the same name
-func (collection *PublishedRepoCollection) CheckDuplicate(repo *PublishedRepo) *PublishedRepo {
-	for _, r := range collection.list {
-		if r.Prefix == repo.Prefix && r.Distribution == repo.Distribution {
-			return r
-		}
-	}
-
-	return nil
-}
-
-// Update stores updated information about repo in DB
-func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) error {
-	err := collection.db.Put(repo.Key(), repo.Encode())
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// LoadComplete loads additional information for remote repo
-func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, snapshotCollection *SnapshotCollection) error {
-	snapshot, err := snapshotCollection.ByUUID(repo.SnapshotUUID)
-	if err != nil {
-		return err
-	}
-
-	repo.snapshot = snapshot
-	return nil
-}
-
-// ByPrefixDistribution looks up repository by prefix & distribution
-func (collection *PublishedRepoCollection) ByPrefixDistribution(prefix, distribution string) (*PublishedRepo, error) {
-	for _, r := range collection.list {
-		if r.Prefix == prefix && r.Distribution == distribution {
-			return r, nil
-		}
-	}
-	return nil, fmt.Errorf("published repo with prefix/distribution %s/%s not found", prefix, distribution)
-}
-
-// ByUUID looks up repository by uuid
-func (collection *PublishedRepoCollection) ByUUID(uuid string) (*PublishedRepo, error) {
-	for _, r := range collection.list {
-		if r.UUID == uuid {
-			return r, nil
-		}
-	}
-	return nil, fmt.Errorf("published repo with uuid %s not found", uuid)
-}
-
-// BySnapshot looks up repository by snapshot source
-func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*PublishedRepo {
-	result := make([]*PublishedRepo, 0)
-	for _, r := range collection.list {
-		if r.SnapshotUUID == snapshot.UUID {
-			result = append(result, r)
-		}
-	}
-	return result
-}
-
-// ForEach runs method for each repository
-func (collection *PublishedRepoCollection) ForEach(handler func(*PublishedRepo) error) error {
-	var err error
-	for _, r := range collection.list {
-		err = handler(r)
-		if err != nil {
-			return err
-		}
-	}
-	return err
-}
-
-// Len returns number of remote repos
-func (collection *PublishedRepoCollection) Len() int {
-	return len(collection.list)
-}
-
-// Remove removes published repository, cleaning up directories, files
-func (collection *PublishedRepoCollection) Remove(publishedStorage aptly.PublishedStorage, prefix, distribution string) error {
-	repo, err := collection.ByPrefixDistribution(prefix, distribution)
-	if err != nil {
-		return err
-	}
-
-	removePrefix := true
-	removePoolComponent := true
-	repoPosition := -1
-
-	for i, r := range collection.list {
-		if r == repo {
-			repoPosition = i
-			continue
-		}
-		if r.Prefix == repo.Prefix {
-			removePrefix = false
-			if r.Component == repo.Component {
-				removePoolComponent = false
-			}
-		}
-	}
-
-	err = repo.RemoveFiles(publishedStorage, removePrefix, removePoolComponent)
-	if err != nil {
-		return err
-	}
-
-	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
-		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
-
-	return collection.db.Delete(repo.Key())
-}

debian/publish_test.go

@@ -1,486 +0,0 @@
-package debian
-
-import (
-	"errors"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/files"
-	. "launchpad.net/gocheck"
-	"os"
-	"path/filepath"
-)
-
-type pathExistsChecker struct {
-	*CheckerInfo
-}
-
-var PathExists = &pathExistsChecker{
-	&CheckerInfo{Name: "PathExists", Params: []string{"path"}},
-}
-
-func (checker *pathExistsChecker) Check(params []interface{}, names []string) (result bool, error string) {
-	_, err := os.Stat(params[0].(string))
-	return err == nil, ""
-}
-
-type NullSigner struct{}
-
-func (n *NullSigner) Init() error {
-	return nil
-}
-
-func (n *NullSigner) SetKey(keyRef string) {
-}
-
-func (n *NullSigner) SetKeyRing(keyring, secretKeyring string) {
-}
-
-func (n *NullSigner) DetachedSign(source string, destination string) error {
-	return nil
-}
-
-func (n *NullSigner) ClearSign(source string, destination string) error {
-	return nil
-}
-
-type PublishedRepoSuite struct {
-	PackageListMixinSuite
-	repo              *PublishedRepo
-	root              string
-	publishedStorage  aptly.PublishedStorage
-	packagePool       aptly.PackagePool
-	snapshot          *Snapshot
-	db                database.Storage
-	packageCollection *PackageCollection
-}
-
-var _ = Suite(&PublishedRepoSuite{})
-
-func (s *PublishedRepoSuite) SetUpTest(c *C) {
-	s.SetUpPackages()
-
-	s.db, _ = database.OpenDB(c.MkDir())
-
-	s.root = c.MkDir()
-	s.publishedStorage = files.NewPublishedStorage(s.root)
-	s.packagePool = files.NewPackagePool(s.root)
-
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
-	repo.packageRefs = s.reflist
-
-	s.snapshot, _ = NewSnapshotFromRepository("snap", repo)
-
-	s.repo, _ = NewPublishedRepo("ppa", "squeeze", "main", nil, s.snapshot)
-
-	s.packageCollection = NewPackageCollection(s.db)
-	s.packageCollection.Update(s.p1)
-	s.packageCollection.Update(s.p2)
-	s.packageCollection.Update(s.p3)
-
-	poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
-	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
-	f, err := os.Create(poolPath)
-	c.Assert(err, IsNil)
-	f.Close()
-}
-
-func (s *PublishedRepoSuite) TearDownTest(c *C) {
-	s.db.Close()
-}
-
-func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
-
-	for _, t := range []struct {
-		prefix        string
-		expected      string
-		errorExpected string
-	}{
-		{
-			prefix:   "ppa",
-			expected: "ppa",
-		},
-		{
-			prefix:   "",
-			expected: ".",
-		},
-		{
-			prefix:   "/",
-			expected: ".",
-		},
-		{
-			prefix:   "//",
-			expected: ".",
-		},
-		{
-			prefix:   "//ppa/",
-			expected: "ppa",
-		},
-		{
-			prefix:   "ppa/..",
-			expected: ".",
-		},
-		{
-			prefix:   "ppa/ubuntu/",
-			expected: "ppa/ubuntu",
-		},
-		{
-			prefix:   "ppa/../ubuntu/",
-			expected: "ubuntu",
-		},
-		{
-			prefix:        "../ppa/",
-			errorExpected: "invalid prefix .*",
-		},
-		{
-			prefix:        "../ppa/../ppa/",
-			errorExpected: "invalid prefix .*",
-		},
-		{
-			prefix:        "ppa/dists",
-			errorExpected: "invalid prefix .*",
-		},
-		{
-			prefix:        "ppa/pool",
-			errorExpected: "invalid prefix .*",
-		},
-	} {
-		repo, err := NewPublishedRepo(t.prefix, "squeeze", "main", nil, s.snapshot)
-		if t.errorExpected != "" {
-			c.Check(err, ErrorMatches, t.errorExpected)
-		} else {
-			c.Check(repo.Prefix, Equals, t.expected)
-		}
-	}
-}
-
-func (s *PublishedRepoSuite) TestPublish(c *C) {
-	err := s.repo.Publish(s.packagePool, s.publishedStorage, s.packageCollection, &NullSigner{}, nil)
-	c.Assert(err, IsNil)
-
-	c.Check(s.repo.Architectures, DeepEquals, []string{"i386"})
-
-	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
-	c.Assert(err, IsNil)
-
-	cfr := NewControlFileReader(rf)
-	st, err := cfr.ReadStanza()
-	c.Assert(err, IsNil)
-
-	c.Check(st["Origin"], Equals, "ppa squeeze")
-	c.Check(st["Components"], Equals, "main")
-	c.Check(st["Architectures"], Equals, "i386")
-
-	pf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages"))
-	c.Assert(err, IsNil)
-
-	cfr = NewControlFileReader(pf)
-
-	for i := 0; i < 3; i++ {
-		st, err = cfr.ReadStanza()
-		c.Assert(err, IsNil)
-
-		c.Check(st["Filename"], Equals, "pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
-	}
-
-	st, err = cfr.ReadStanza()
-	c.Assert(err, IsNil)
-	c.Assert(st, IsNil)
-
-	_, err = os.Stat(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb"))
-	c.Assert(err, IsNil)
-}
-
-func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
-	err := s.repo.Publish(s.packagePool, s.publishedStorage, s.packageCollection, nil, nil)
-	c.Assert(err, IsNil)
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists)
-}
-
-func (s *PublishedRepoSuite) TestString(c *C) {
-	c.Check(s.repo.String(), Equals,
-		"ppa/squeeze (main) publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
-	repo, _ := NewPublishedRepo("", "squeeze", "main", nil, s.snapshot)
-	c.Check(repo.String(), Equals,
-		"./squeeze (main) publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
-	repo, _ = NewPublishedRepo("", "squeeze", "main", []string{"i386", "amd64"}, s.snapshot)
-	c.Check(repo.String(), Equals,
-		"./squeeze (main) [i386, amd64] publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
-}
-
-func (s *PublishedRepoSuite) TestKey(c *C) {
-	c.Check(s.repo.Key(), DeepEquals, []byte("Uppa>>squeeze"))
-}
-
-func (s *PublishedRepoSuite) TestEncodeDecode(c *C) {
-	encoded := s.repo.Encode()
-	repo := &PublishedRepo{}
-	err := repo.Decode(encoded)
-
-	s.repo.snapshot = nil
-	c.Assert(err, IsNil)
-	c.Assert(repo, DeepEquals, s.repo)
-}
-
-type PublishedRepoCollectionSuite struct {
-	PackageListMixinSuite
-	db                  database.Storage
-	snapshotCollection  *SnapshotCollection
-	collection          *PublishedRepoCollection
-	snap1, snap2        *Snapshot
-	repo1, repo2, repo3 *PublishedRepo
-}
-
-var _ = Suite(&PublishedRepoCollectionSuite{})
-
-func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
-
-	s.snapshotCollection = NewSnapshotCollection(s.db)
-
-	s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
-	s.snap2 = NewSnapshotFromPackageList("snap2", []*Snapshot{}, NewPackageList(), "desc2")
-
-	s.snapshotCollection.Add(s.snap1)
-	s.snapshotCollection.Add(s.snap2)
-
-	s.repo1, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap1)
-	s.repo2, _ = NewPublishedRepo("", "anaconda", "main", []string{}, s.snap2)
-	s.repo3, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap2)
-
-	s.collection = NewPublishedRepoCollection(s.db)
-}
-
-func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) {
-	s.db.Close()
-}
-
-func (s *PublishedRepoCollectionSuite) TestAddByPrefixDistribution(c *C) {
-	r, err := s.collection.ByPrefixDistribution("ppa", "anaconda")
-	c.Assert(err, ErrorMatches, "*.not found")
-
-	c.Assert(s.collection.Add(s.repo1), IsNil)
-	c.Assert(s.collection.Add(s.repo1), ErrorMatches, ".*already exists")
-	c.Assert(s.collection.CheckDuplicate(s.repo2), IsNil)
-	c.Assert(s.collection.Add(s.repo2), IsNil)
-	c.Assert(s.collection.Add(s.repo3), ErrorMatches, ".*already exists")
-	c.Assert(s.collection.CheckDuplicate(s.repo3), Equals, s.repo1)
-
-	r, err = s.collection.ByPrefixDistribution("ppa", "anaconda")
-	c.Assert(err, IsNil)
-
-	err = s.collection.LoadComplete(r, s.snapshotCollection)
-	c.Assert(err, IsNil)
-	c.Assert(r.String(), Equals, s.repo1.String())
-
-	collection := NewPublishedRepoCollection(s.db)
-	r, err = collection.ByPrefixDistribution("ppa", "anaconda")
-	c.Assert(err, IsNil)
-
-	err = s.collection.LoadComplete(r, s.snapshotCollection)
-	c.Assert(err, IsNil)
-	c.Assert(r.String(), Equals, s.repo1.String())
-}
-
-func (s *PublishedRepoCollectionSuite) TestByUUID(c *C) {
-	r, err := s.collection.ByUUID(s.repo1.UUID)
-	c.Assert(err, ErrorMatches, "*.not found")
-
-	c.Assert(s.collection.Add(s.repo1), IsNil)
-
-	r, err = s.collection.ByUUID(s.repo1.UUID)
-	c.Assert(err, IsNil)
-
-	err = s.collection.LoadComplete(r, s.snapshotCollection)
-	c.Assert(err, IsNil)
-	c.Assert(r.String(), Equals, s.repo1.String())
-}
-
-func (s *PublishedRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
-	c.Assert(s.collection.Update(s.repo1), IsNil)
-
-	collection := NewPublishedRepoCollection(s.db)
-	r, err := collection.ByPrefixDistribution("ppa", "anaconda")
-	c.Assert(err, IsNil)
-	c.Assert(r.snapshot, IsNil)
-	c.Assert(s.collection.LoadComplete(r, s.snapshotCollection), IsNil)
-	c.Assert(r.snapshot.UUID, Equals, s.repo1.snapshot.UUID)
-}
-
-func (s *PublishedRepoCollectionSuite) TestForEachAndLen(c *C) {
-	s.collection.Add(s.repo1)
-
-	count := 0
-	err := s.collection.ForEach(func(*PublishedRepo) error {
-		count++
-		return nil
-	})
-	c.Assert(count, Equals, 1)
-	c.Assert(err, IsNil)
-
-	c.Check(s.collection.Len(), Equals, 1)
-
-	e := errors.New("c")
-
-	err = s.collection.ForEach(func(*PublishedRepo) error {
-		return e
-	})
-	c.Assert(err, Equals, e)
-}
-
-func (s *PublishedRepoCollectionSuite) TestBySnapshot(c *C) {
-	c.Check(s.collection.Add(s.repo1), IsNil)
-	c.Check(s.collection.Add(s.repo2), IsNil)
-
-	c.Check(s.collection.BySnapshot(s.snap1), DeepEquals, []*PublishedRepo{s.repo1})
-	c.Check(s.collection.BySnapshot(s.snap2), DeepEquals, []*PublishedRepo{s.repo2})
-}
-
-type PublishedRepoRemoveSuite struct {
-	PackageListMixinSuite
-	db                         database.Storage
-	snapshotCollection         *SnapshotCollection
-	collection                 *PublishedRepoCollection
-	root                       string
-	publishedStorage           aptly.PublishedStorage
-	snap1                      *Snapshot
-	repo1, repo2, repo3, repo4 *PublishedRepo
-}
-
-var _ = Suite(&PublishedRepoRemoveSuite{})
-
-func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
-
-	s.snapshotCollection = NewSnapshotCollection(s.db)
-
-	s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
-
-	s.snapshotCollection.Add(s.snap1)
-
-	s.repo1, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap1)
-	s.repo2, _ = NewPublishedRepo("", "anaconda", "main", []string{}, s.snap1)
-	s.repo3, _ = NewPublishedRepo("ppa", "meduza", "main", []string{}, s.snap1)
-	s.repo4, _ = NewPublishedRepo("ppa", "osminog", "contrib", []string{}, s.snap1)
-
-	s.collection = NewPublishedRepoCollection(s.db)
-	s.collection.Add(s.repo1)
-	s.collection.Add(s.repo2)
-	s.collection.Add(s.repo3)
-	s.collection.Add(s.repo4)
-
-	s.root = c.MkDir()
-	s.publishedStorage = files.NewPublishedStorage(s.root)
-	s.publishedStorage.MkDir("ppa/dists/anaconda")
-	s.publishedStorage.MkDir("ppa/dists/meduza")
-	s.publishedStorage.MkDir("ppa/dists/osminog")
-	s.publishedStorage.MkDir("ppa/pool/main")
-	s.publishedStorage.MkDir("ppa/pool/contrib")
-	s.publishedStorage.MkDir("dists/anaconda")
-	s.publishedStorage.MkDir("pool/main")
-}
-
-func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
-	s.db.Close()
-}
-
-func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
-	s.repo1.RemoveFiles(s.publishedStorage, false, false)
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
-}
-
-func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
-	s.repo1.RemoveFiles(s.publishedStorage, false, true)
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
-}
-
-func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
-	s.repo1.RemoveFiles(s.publishedStorage, true, true)
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
-}
-
-func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
-	s.repo2.RemoveFiles(s.publishedStorage, true, true)
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), Not(PathExists))
-}
-
-func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
-	err := s.collection.Remove(s.publishedStorage, "ppa", "anaconda")
-	c.Check(err, IsNil)
-
-	_, err = s.collection.ByPrefixDistribution("ppa", "anaconda")
-	c.Check(err, ErrorMatches, ".*not found")
-
-	collection := NewPublishedRepoCollection(s.db)
-	_, err = collection.ByPrefixDistribution("ppa", "anaconda")
-	c.Check(err, ErrorMatches, ".*not found")
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
-
-	err = s.collection.Remove(s.publishedStorage, "ppa", "anaconda")
-	c.Check(err, ErrorMatches, ".*not found")
-
-	err = s.collection.Remove(s.publishedStorage, "ppa", "meduza")
-	c.Check(err, IsNil)
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
-}
-
-func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
-	err := s.collection.Remove(s.publishedStorage, ".", "anaconda")
-	c.Check(err, IsNil)
-
-	_, err = s.collection.ByPrefixDistribution(".", "anaconda")
-	c.Check(err, ErrorMatches, ".*not found")
-
-	collection := NewPublishedRepoCollection(s.db)
-	_, err = collection.ByPrefixDistribution(".", "anaconda")
-	c.Check(err, ErrorMatches, ".*not found")
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/"), Not(PathExists))
-}

files/public.go

@@ -3,9 +3,10 @@ package files
 import (
 	"fmt"
 	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/utils"
+	"io"
 	"os"
 	"path/filepath"
+	"syscall"
 )
 
 // PublishedStorage abstract file system with public dirs (published repos)
@@ -13,9 +14,10 @@ type PublishedStorage struct {
 	rootPath string
 }
 
-// Check interface
+// Check interfaces
 var (
-	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+	_ aptly.PublishedStorage      = (*PublishedStorage)(nil)
+	_ aptly.LocalPublishedStorage = (*PublishedStorage)(nil)
 )
 
 // NewPublishedStorage creates new instance of PublishedStorage which specified root
@@ -33,51 +35,109 @@ func (storage *PublishedStorage) MkDir(path string) error {
 	return os.MkdirAll(filepath.Join(storage.rootPath, path), 0755)
 }
 
-// CreateFile creates file for writing under public path
-func (storage *PublishedStorage) CreateFile(path string) (*os.File, error) {
-	return os.Create(filepath.Join(storage.rootPath, path))
+// PutFile puts file into published storage at specified path
+func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
+	var (
+		source, f *os.File
+		err       error
+	)
+	source, err = os.Open(sourceFilename)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	f, err = os.Create(filepath.Join(storage.rootPath, path))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = io.Copy(f, source)
+	return err
+}
+
+// Remove removes single file under public path
+func (storage *PublishedStorage) Remove(path string) error {
+	filepath := filepath.Join(storage.rootPath, path)
+	return os.Remove(filepath)
 }
 
 // RemoveDirs removes directory structure under public path
-func (storage *PublishedStorage) RemoveDirs(path string) error {
+func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
 	filepath := filepath.Join(storage.rootPath, path)
-	fmt.Printf("Removing %s...\n", filepath)
+	if progress != nil {
+		progress.Printf("Removing %s...\n", filepath)
+	}
 	return os.RemoveAll(filepath)
 }
 
 // LinkFromPool links package file from pool to dist's pool location
 //
-// prefix is publishing prefix for this repo (e.g. empty or "ppa/")
-// component is component name when publishing (e.g. main)
-// poolDirectory is desired location in pool (like liba/libav/)
+// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
 // sourcePool is instance of aptly.PackagePool
 // sourcePath is filepath to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(prefix string, component string, poolDirectory string, sourcePool aptly.PackagePool, sourcePath string) (string, error) {
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool, sourcePath, sourceMD5 string) error {
 	// verify that package pool is local pool is filesystem pool
 	_ = sourcePool.(*PackagePool)
 
 	baseName := filepath.Base(sourcePath)
-
-	relPath := filepath.Join("pool", component, poolDirectory, baseName)
-	poolPath := filepath.Join(storage.rootPath, prefix, "pool", component, poolDirectory)
+	poolPath := filepath.Join(storage.rootPath, publishedDirectory)
 
 	err := os.MkdirAll(poolPath, 0755)
 	if err != nil {
-		return "", err
+		return err
 	}
 
-	_, err = os.Stat(filepath.Join(poolPath, baseName))
-	if err == nil { // already exists, skip
-		return relPath, nil
+	var dstStat, srcStat os.FileInfo
+
+	dstStat, err = os.Stat(filepath.Join(poolPath, baseName))
+	if err == nil {
+		// already exists, check source file
+		srcStat, err = os.Stat(sourcePath)
+		if err != nil {
+			// source file doesn't exist? problem!
+			return err
+		}
+
+		srcSys := srcStat.Sys().(*syscall.Stat_t)
+		dstSys := dstStat.Sys().(*syscall.Stat_t)
+
+		if srcSys.Ino != dstSys.Ino {
+			return fmt.Errorf("error linking file to %s: file already exists and is different", filepath.Join(poolPath, baseName))
+		}
+		return nil
 	}
 
-	err = os.Link(sourcePath, filepath.Join(poolPath, baseName))
-	return relPath, err
+	return os.Link(sourcePath, filepath.Join(poolPath, baseName))
 }
 
-// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
-func (storage *PublishedStorage) ChecksumsForFile(path string) (utils.ChecksumInfo, error) {
-	return utils.ChecksumsForFile(filepath.Join(storage.rootPath, path))
+// Filelist returns list of files under prefix
+func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
+	root := filepath.Join(storage.rootPath, prefix)
+	result := []string{}
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			result = append(result, path[len(root)+1:])
+		}
+		return nil
+	})
+
+	if err != nil && os.IsNotExist(err) {
+		// file path doesn't exist, consider it empty
+		return []string{}, nil
+	}
+
+	return result, err
+}
+
+// RenameFile renames (moves) file
+func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
+	return os.Rename(filepath.Join(storage.rootPath, oldName), filepath.Join(storage.rootPath, newName))
 }

files/public_test.go

@@ -32,27 +32,72 @@ func (s *PublishedStorageSuite) TestMkDir(c *C) {
 	c.Assert(err, IsNil)
 }
 
-func (s *PublishedStorageSuite) TestCreateFile(c *C) {
+func (s *PublishedStorageSuite) TesPutFile(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
-	file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file.Close()
 
 	_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
 	c.Assert(err, IsNil)
 }
 
+func (s *PublishedStorageSuite) TestFilelist(c *C) {
+	err := s.storage.MkDir("ppa/pool/main/a/ab/")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/pool/main/a/ab/a.deb", "/dev/null")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/pool/main/a/ab/b.deb", "/dev/null")
+	c.Assert(err, IsNil)
+
+	list, err := s.storage.Filelist("ppa/pool/main/")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a/ab/a.deb", "a/ab/b.deb"})
+
+	list, err = s.storage.Filelist("ppa/pool/doenstexist/")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+}
+
+func (s *PublishedStorageSuite) TestRenameFile(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	err = s.storage.RenameFile("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease")
+	c.Check(err, IsNil)
+
+	_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/InRelease"))
+	c.Assert(err, IsNil)
+}
+
 func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
-	file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
 	c.Assert(err, IsNil)
-	defer file.Close()
 
-	err = s.storage.RemoveDirs("ppa/dists/")
+	err = s.storage.RemoveDirs("ppa/dists/", nil)
+
+	_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
+	c.Assert(err, NotNil)
+	c.Assert(os.IsNotExist(err), Equals, true)
+}
+
+func (s *PublishedStorageSuite) TestRemove(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	err = s.storage.Remove("ppa/dists/squeeze/Release")
 
 	_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
 	c.Assert(err, NotNil)
@@ -108,9 +153,8 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		err = ioutil.WriteFile(t.sourcePath, []byte("Contents"), 0644)
 		c.Assert(err, IsNil)
 
-		path, err := s.storage.LinkFromPool(t.prefix, t.component, t.poolDirectory, pool, t.sourcePath)
+		err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, "")
 		c.Assert(err, IsNil)
-		c.Assert(path, Equals, t.expectedFilename)
 
 		st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename))
 		c.Assert(err, IsNil)
@@ -118,4 +162,15 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		info := st.Sys().(*syscall.Stat_t)
 		c.Check(int(info.Nlink), Equals, 2)
 	}
+
+	// test linking files to duplicate final name
+	sourcePath := filepath.Join(s.root, "pool/02/bc/mars-invaders_1.03.deb")
+	err := os.MkdirAll(filepath.Dir(sourcePath), 0755)
+	c.Assert(err, IsNil)
+
+	err = ioutil.WriteFile(sourcePath, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "")
+	c.Check(err, ErrorMatches, ".*file already exists and is different")
 }

http/download.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"code.google.com/p/mxk/go1/flowcontrol"
 	"compress/bzip2"
 	"compress/gzip"
 	"fmt"
@@ -21,13 +22,15 @@ var (
 
 // downloaderImpl is implementation of Downloader interface
 type downloaderImpl struct {
-	queue    chan *downloadTask
-	stop     chan bool
-	stopped  chan bool
-	pause    chan bool
-	unpause  chan bool
-	progress aptly.Progress
-	threads  int
+	queue     chan *downloadTask
+	stop      chan struct{}
+	stopped   chan struct{}
+	pause     chan struct{}
+	unpause   chan struct{}
+	progress  aptly.Progress
+	aggWriter io.Writer
+	threads   int
+	client    *http.Client
 }
 
 // downloadTask represents single item in queue
@@ -40,16 +43,28 @@ type downloadTask struct {
 }
 
 // NewDownloader creates new instance of Downloader which specified number
-// of threads
-func NewDownloader(threads int, progress aptly.Progress) aptly.Downloader {
+// of threads and download limit in bytes/sec
+func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.Downloader {
+	transport := *http.DefaultTransport.(*http.Transport)
+	transport.DisableCompression = true
+
 	downloader := &downloaderImpl{
 		queue:    make(chan *downloadTask, 1000),
-		stop:     make(chan bool),
-		stopped:  make(chan bool),
-		pause:    make(chan bool),
-		unpause:  make(chan bool),
+		stop:     make(chan struct{}),
+		stopped:  make(chan struct{}),
+		pause:    make(chan struct{}),
+		unpause:  make(chan struct{}),
 		threads:  threads,
 		progress: progress,
+		client: &http.Client{
+			Transport: &transport,
+		},
+	}
+
+	if downLimit > 0 {
+		downloader.aggWriter = flowcontrol.NewWriter(progress, downLimit)
+	} else {
+		downloader.aggWriter = progress
 	}
 
 	for i := 0; i < downloader.threads; i++ {
@@ -63,7 +78,7 @@ func NewDownloader(threads int, progress aptly.Progress) aptly.Downloader {
 // but doesn't process rest of queue
 func (downloader *downloaderImpl) Shutdown() {
 	for i := 0; i < downloader.threads; i++ {
-		downloader.stop <- true
+		downloader.stop <- struct{}{}
 	}
 
 	for i := 0; i < downloader.threads; i++ {
@@ -74,14 +89,14 @@ func (downloader *downloaderImpl) Shutdown() {
 // Pause pauses task processing
 func (downloader *downloaderImpl) Pause() {
 	for i := 0; i < downloader.threads; i++ {
-		downloader.pause <- true
+		downloader.pause <- struct{}{}
 	}
 }
 
 // Resume resumes task processing
 func (downloader *downloaderImpl) Resume() {
 	for i := 0; i < downloader.threads; i++ {
-		downloader.unpause <- true
+		downloader.unpause <- struct{}{}
 	}
 }
 
@@ -105,7 +120,7 @@ func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination s
 func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	downloader.progress.Printf("Downloading %s...\n", task.url)
 
-	resp, err := http.Get(task.url)
+	resp, err := downloader.client.Get(task.url)
 	if err != nil {
 		task.result <- err
 		return
@@ -133,7 +148,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	defer outfile.Close()
 
 	checksummer := utils.NewChecksumWriter()
-	writers := []io.Writer{outfile, downloader.progress}
+	writers := []io.Writer{outfile, downloader.aggWriter}
 
 	if task.expected.Size != -1 {
 		writers = append(writers, checksummer)
@@ -187,7 +202,7 @@ func (downloader *downloaderImpl) process() {
 	for {
 		select {
 		case <-downloader.stop:
-			downloader.stopped <- true
+			downloader.stopped <- struct{}{}
 			return
 		case <-downloader.pause:
 			<-downloader.unpause

http/download_test.go

@@ -60,7 +60,7 @@ func (s *DownloaderSuite) TearDownTest(c *C) {
 func (s *DownloaderSuite) TestStartupShutdown(c *C) {
 	goroutines := runtime.NumGoroutine()
 
-	d := NewDownloader(10, s.progress)
+	d := NewDownloader(10, 100, s.progress)
 	d.Shutdown()
 
 	// wait for goroutines to shutdown
@@ -72,7 +72,7 @@ func (s *DownloaderSuite) TestStartupShutdown(c *C) {
 }
 
 func (s *DownloaderSuite) TestPauseResume(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 
 	d.Pause()
@@ -80,7 +80,7 @@ func (s *DownloaderSuite) TestPauseResume(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadOK(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -90,7 +90,7 @@ func (s *DownloaderSuite) TestDownloadOK(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -131,7 +131,7 @@ func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownload404(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -141,7 +141,7 @@ func (s *DownloaderSuite) TestDownload404(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadConnectError(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -151,7 +151,7 @@ func (s *DownloaderSuite) TestDownloadConnectError(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadFileError(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 	ch := make(chan error)
 
@@ -161,7 +161,7 @@ func (s *DownloaderSuite) TestDownloadFileError(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadTemp(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 
 	f, err := DownloadTemp(d, s.url+"/test")
@@ -178,7 +178,7 @@ func (s *DownloaderSuite) TestDownloadTemp(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadTempWithChecksum(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 
 	f, err := DownloadTempWithChecksum(d, s.url+"/test", utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
@@ -191,7 +191,7 @@ func (s *DownloaderSuite) TestDownloadTempWithChecksum(c *C) {
 }
 
 func (s *DownloaderSuite) TestDownloadTempError(c *C) {
-	d := NewDownloader(2, s.progress)
+	d := NewDownloader(2, 0, s.progress)
 	defer d.Shutdown()
 
 	f, err := DownloadTemp(d, s.url+"/doesntexist")

main.go

@@ -2,96 +2,37 @@ package main
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
 	"github.com/smira/aptly/cmd"
-	"github.com/smira/aptly/utils"
 	"os"
-	"path/filepath"
 )
 
-var (
-	returnCode   = 0
-	errorMessage string
-)
-
-func fatal(err error) {
-	errorMessage = fmt.Sprintf("ERROR: %s\n", err)
-	returnCode = 1
-}
-
-func loadConfig(command *commander.Command) error {
-	var err error
-
-	configLocation := command.Flag.Lookup("config").Value.String()
-	if configLocation != "" {
-		err = utils.LoadConfig(configLocation, &utils.Config)
-
-		if err != nil {
-			return err
-		}
-	} else {
-		configLocations := []string{
-			filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
-			"/etc/aptly.conf",
-		}
-
-		for _, configLocation := range configLocations {
-			err = utils.LoadConfig(configLocation, &utils.Config)
-			if err == nil {
-				break
-			}
-			if !os.IsNotExist(err) {
-				fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
-				return nil
-			}
-		}
-
-		if err != nil {
-			fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
-			utils.SaveConfig(configLocations[0], &utils.Config)
-		}
-	}
-
-	return nil
-}
-
 func main() {
 	defer func() {
-		if errorMessage != "" {
-			fmt.Print(errorMessage)
-		}
-		if returnCode != 0 {
-			os.Exit(returnCode)
+		if r := recover(); r != nil {
+			fatal, ok := r.(*cmd.FatalError)
+			if !ok {
+				panic(r)
+			}
+			fmt.Println("ERROR:", fatal.Message)
+			os.Exit(fatal.ReturnCode)
 		}
 	}()
 
 	command := cmd.RootCommand()
 
-	err := command.Flag.Parse(os.Args[1:])
+	flags, args, err := command.ParseFlags(os.Args[1:])
 	if err != nil {
-		fatal(err)
-		return
+		cmd.Fatal(err)
 	}
 
-	err = loadConfig(command)
+	err = cmd.InitContext(flags)
 	if err != nil {
-		fatal(err)
-		return
-	}
-	if returnCode != 0 {
-		return
-	}
-
-	err = cmd.InitContext(command)
-	if err != nil {
-		fatal(err)
-		return
+		cmd.Fatal(err)
 	}
 	defer cmd.ShutdownContext()
 
-	err = command.Dispatch(command.Flag.Args())
+	err = command.Dispatch(args)
 	if err != nil {
-		fatal(err)
-		return
+		cmd.Fatal(err)
 	}
 }

man/aptly.1.ronn.tmpl

@@ -28,6 +28,7 @@ Configuration file is stored in JSON format (default values shown below):
     {
       "rootDir": "$HOME/.aptly",
       "downloadConcurrency": 4,
+      "downloadSpeedLimit": 0,
       "architectures": [],
       "dependencyFollowSuggests": false,
       "dependencyFollowRecommends": false
@@ -37,7 +38,16 @@ Configuration file is stored in JSON format (default values shown below):
       "gpgDisableVerify": false,
       "downloadSourcePackages": false,
       "ppaDistributorID": "ubuntu",
-      "ppaCodename": ""
+      "ppaCodename": "",
+      "S3PublishEndpoints": {
+        "test": {
+          "region": "us-east-1",
+          "bucket": "repo",
+          "awsAccessKeyID": ""
+          "awsSecretAccessKey": "",
+          "prefix": "",
+          "acl": "public-read"
+        }
     }
 
 Options:
@@ -49,6 +59,9 @@ Options:
   * `downloadConcurrency`:
     is a number of parallel download threads to use when downloading packages
 
+  * `downloadSpeedLimit`:
+    limit in kbytes/sec on download speed while mirroring remote repositieis
+
   * `architectures`:
     is a list of architectures to process; if left empty defaults to all available architectures; could be
     overridden with option `-architectures`
@@ -81,10 +94,43 @@ Options:
     specifies paramaters for short PPA url expansion, if left blank they default
     to output of `lsb_release` command
 
-## PACKAGE SPEC
+  * `S3PublisEndpoints`:
+    configuration of Amazon S3 publishing endpoints (see below)
 
-Some commands accept package specs to identify list of packages to process.
-Package spec is a list of following search conditions:
+## S3 PUBLISHING ENDPOINTS
+
+aptly could be configured to publish repository directly to Amazon S3. First, publishing
+endpoints should be described in aptly configuration file. Each endpoint has name
+and associated settings:
+
+   * `region`:
+     Amazon region for S3 bucket (e.g. `us-east-1`)
+   * `bucket`:
+     bucket name
+   * `prefix`:
+     (optional) do publishing under specified prefix in the bucket, defaults to
+     no prefix (bucket root)
+   * `acl`:
+     (optional) assign ACL to published files (one of the canned ACLs in Amazon
+     terminology). Useful values: `private` (default) or `public-read` (public
+     repository). Public repositories could be consumed by `apt` using
+     HTTP endpoint (Amazon bucket should be configured for "website hosting"),
+     for private repositories special apt S3 transport is required.
+   * `awsAccessKeyID`, `awsSecretAccessKey`:
+     (optional) Amazon credentials to access S3 bucket. If not supplied,
+     environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+     are used.
+
+In order to publish to S3, specify endpoint as `s3:endpoint-name:` before
+publishing prefix on the command line, e.g.:
+
+  `aptly publish snapshot wheezy-main s3:test:`
+
+## PACKAGE QUERY
+
+Some commands accept package queries to identify list of packages to process.
+Package query syntax almost matches `reprepro` query language. Query consists of
+the following simple terms:
 
   * direct package reference:
     reference to exaclty one package. Format is identical to the way aptly lists packages in
@@ -92,12 +138,49 @@ Package spec is a list of following search conditions:
     e.g.: `libmysqlclient18_5.5.35-rel33.0-611.squeeze_amd64`
 
   * dependency condition:
-    syntax follows Debian dependency specification: package_name followed by optional version specification and architecture limit.
+    syntax follows Debian dependency specification: package_name followed by optional version specification
+    and architecture limit, e.g: `mysql-client (>= 3.6)`.
+
+  * query against package fields:
+    syntax is the same as for dependency conditions, but instead of package name field name is used, e.g:
+    `Priority (optional)`.
+
+Supported fields:
+
+  * all field names from Debian package control files are supported except for `Filename`, `MD5sum`,
+    `SHA1`, `SHA256`, `Size`, `Files`, `Checksums-SHA1`, `Checksums-SHA256`.
+  * `$Source` is a name of source package (for binary packages)
+  * `$SourceVersion` is a version of source package
+  * `$Architecture` is `Architecture` for binary packages and `source` for source packages,
+     when matching with equal (`=`) operator, package with `any` architecture matches all architectures
+     but `source`.
+  * `$Version` has the same value as `Version`, but comparison operators use Debian
+     version precedence rules
+  * `$PackageType` is `deb` for binary packages and `source` for source packages
+
+Operators:
+
+  * `=`:
+    strict match, default operator is no operator is given
+  * `>=`, `<=`, `=`, `>>` (strictly greater), `<<` (strictly less):
+    lexicographical comparison for all fields and special rules when comparing package versions
+  * `%`:
+    pattern matching, like shell patterns, supported special symbols are: `[^]?*`, e.g.:
+    `$Version (% 3.5-*)`
+  * `~`:
+    regular expression matching, e.g.:
+    `Name (~ .*-dev)`
+
+Simple terms could be combined into more complex queries using operators `,` (and), `|` (or) and
+`!` (not), parentheses `()` are used to change operator precedence. Match value could be
+enclosed in single (`'`) or double (`"`) quotes if required to resolve ambiguity, quotes
+inside quoted string should escaped with slash (`\`).
 
 Examples:
 
   * `mysql-client`:
-     matches package mysql-client of any version and architecture (including source)
+     matches package mysql-client of any version and architecture (including source), also
+     matches packages that `Provide:` `mysql-client`.
 
   * `mysql-client (>= 3.6)`:
      matches package mysql-client with version greater or equal to 3.6. Valid operators for
@@ -107,9 +190,19 @@ Examples:
      matches package `mysql-client` on architecture `i386`, architecture `all` matches all architectures but source.
 
   * `mysql-client (>= 3.6) {i386}`:
-      version and architecture conditions combined.
+    version and architecture conditions combined.
 
-When specified on command line, condition may have to be quoted according to shell rules, so that it stays single argument:
+  * `libmysqlclient18_5.5.35-rel33.0-611.squeeze_amd64`:
+    direct package reference.
+
+  * `$Source (nginx)`:
+    all binary packages with `nginx` as source package.
+
+  * `!Name (~ .*-dev), mail-transport, $Version (>= 3.5)`:
+    matches all packages that provide `mail-transport` with name that has no suffix `-dev` and
+    with version greater or equal to `3.5`.
+
+When specified on command line, query may have to be quoted according to shell rules, so that it stays single argument:
 
   `aptly repo import percona stable 'mysql-client (>= 3.6)'`
 
@@ -139,7 +232,16 @@ to proxy all HTTP requests.
 
 ## RETURN VALUES
 
-`aptly` exists with 0 on success and with 1 on failure.
+`aptly` exists with:
+
+ * 0:
+   success
+
+ * 1:
+   general failure
+
+ * 2:
+   command parse failure
 
 ## AUTHORS
 

man/gen.go

@@ -2,9 +2,9 @@ package main
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
 	"github.com/smira/aptly/cmd"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"log"
 	"os"
 	"os/exec"

query/lex.go

@@ -0,0 +1,247 @@
+package query
+
+import (
+	"fmt"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// itemType identifies the type of lex items.
+type itemType int
+
+const eof = -1
+
+const (
+	itemNull  itemType = iota
+	itemError          // error occurred;
+	// value is text of error
+	itemEOF
+	itemLeftParen  // (
+	itemRightParen // )
+	itemOr         // |
+	itemAnd        // ,
+	itemNot        // !
+	itemLt         // <<
+	itemLtEq       // <=, <
+	itemGt         // >>
+	itemGtEq       // >=, >
+	itemEq         // =
+	itemPatMatch   // %
+	itemRegexp     // ~
+	itemLeftCurly  // {
+	itemRightCurly // }
+	itemString
+)
+
+// item represents a token returned from the scanner.
+type item struct {
+	typ itemType // Type, such as itemNumber.
+	val string   // Value, such as "23.2".
+}
+
+func (i item) String() string {
+	if i.typ == itemString {
+		return fmt.Sprintf("%#v", i.val)
+	}
+	if i.typ == itemEOF {
+		return "<EOL>"
+	}
+	if i.typ == itemError {
+		return fmt.Sprintf("error: %s", i.val)
+	}
+	if i.typ == itemNull {
+		return "<NULL>"
+	}
+	return i.val
+}
+
+// stateFn represents the state of the scanner
+// as a function that returns the next state.
+type stateFn func(*lexer) stateFn
+
+// lexer holds the state of the scanner.
+type lexer struct {
+	name  string    // used only for error reports.
+	input string    // the string being scanned.
+	start int       // start position of this item.
+	pos   int       // current position in the input.
+	width int       // width of last rune read from input.
+	items chan item // channel of scanned items.
+	last  item
+}
+
+func lex(name, input string) (*lexer, chan item) {
+	l := &lexer{
+		name:  name,
+		input: input,
+		items: make(chan item),
+	}
+	go l.run() // Concurrently run state machine.
+	return l, l.items
+}
+
+// emit passes an item back to the client.
+func (l *lexer) emit(t itemType) {
+	l.items <- item{t, l.input[l.start:l.pos]}
+	l.start = l.pos
+}
+
+// run lexes the input by executing state functions until
+// the state is nil.
+func (l *lexer) run() {
+	for state := lexMain; state != nil; {
+		state = state(l)
+	}
+	close(l.items) // No more tokens will be delivered.
+}
+
+// next returns the next rune in the input.
+func (l *lexer) next() (r rune) {
+	if l.pos >= len(l.input) {
+		l.width = 0
+		return eof
+	}
+	r, l.width =
+		utf8.DecodeRuneInString(l.input[l.pos:])
+	l.pos += l.width
+	return r
+}
+
+// ignore skips over the pending input before this point.
+func (l *lexer) ignore() {
+	l.start = l.pos
+}
+
+// backup steps back one rune.
+// Can be called only once per call of next.
+func (l *lexer) backup() {
+	l.pos -= l.width
+}
+
+// peek returns but does not consume
+// the next rune in the input.
+func (l *lexer) peek() rune {
+	r := l.next()
+	l.backup()
+	return r
+}
+
+func (l *lexer) Current() item {
+	if l.last.typ == 0 {
+		l.last = <-l.items
+	}
+
+	return l.last
+}
+
+func (l *lexer) Consume() {
+	l.last = <-l.items
+}
+
+// error returns an error token and terminates the scan
+// by passing back a nil pointer that will be the next
+// state, terminating l.run.
+func (l *lexer) errorf(format string, args ...interface{}) stateFn {
+	l.items <- item{
+		itemError,
+		fmt.Sprintf(format, args...),
+	}
+	return nil
+}
+
+func lexMain(l *lexer) stateFn {
+	switch r := l.next(); {
+	case r == eof:
+		l.emit(itemEOF)
+		return nil
+	case unicode.IsSpace(r):
+		l.ignore()
+	case r == '(':
+		l.emit(itemLeftParen)
+	case r == ')':
+		l.emit(itemRightParen)
+	case r == '{':
+		l.emit(itemLeftCurly)
+	case r == '}':
+		l.emit(itemRightCurly)
+	case r == '|':
+		l.emit(itemOr)
+	case r == ',':
+		l.emit(itemAnd)
+	case r == '!':
+		l.emit(itemNot)
+	case r == '<':
+		r2 := l.next()
+		if r2 == '<' {
+			l.emit(itemLt)
+		} else if r2 == '=' {
+			l.emit(itemLtEq)
+		} else {
+			l.backup()
+			l.emit(itemLtEq)
+		}
+	case r == '>':
+		r2 := l.next()
+		if r2 == '>' {
+			l.emit(itemGt)
+		} else if r2 == '=' {
+			l.emit(itemGtEq)
+		} else {
+			l.backup()
+			l.emit(itemGtEq)
+		}
+	case r == '=':
+		l.emit(itemEq)
+	case r == '%':
+		l.emit(itemPatMatch)
+	case r == '~':
+		l.emit(itemRegexp)
+	default:
+		l.backup()
+		return lexString
+	}
+
+	return lexMain
+}
+
+func lexString(l *lexer) stateFn {
+	r := l.next()
+	// quoted string
+	if r == '"' || r == '\'' {
+		quote := r
+		result := ""
+		l.ignore()
+		for {
+			r = l.next()
+			if r == quote {
+				l.ignore()
+				l.items <- item{itemString, result}
+				return lexMain
+			}
+			if r == '\\' {
+				r = l.next()
+			}
+			if r == eof {
+				return l.errorf("unexpected eof in quoted string")
+			}
+			result = result + string(r)
+		}
+	} else {
+		// unquoted string
+		for {
+			if unicode.IsSpace(r) || strings.IndexRune("()|,!{}", r) > 0 {
+				l.backup()
+				l.emit(itemString)
+				return lexMain
+			}
+
+			if r == eof {
+				l.emit(itemString)
+				l.emit(itemEOF)
+				return nil
+			}
+			r = l.next()
+		}
+	}
+}

query/lex_test.go

@@ -0,0 +1,57 @@
+package query
+
+import (
+	"fmt"
+	. "launchpad.net/gocheck"
+)
+
+type LexerSuite struct {
+}
+
+var _ = Suite(&LexerSuite{})
+
+func (s *LexerSuite) TestLexing(c *C) {
+	_, ch := lex("query", "package (<< 1.3), $Source | !\"app\", 'd\"\\a\\'ta' {i386}")
+
+	c.Check(<-ch, Equals, item{typ: itemString, val: "package"})
+	c.Check(<-ch, Equals, item{typ: itemLeftParen, val: "("})
+	c.Check(<-ch, Equals, item{typ: itemLt, val: "<<"})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "1.3"})
+	c.Check(<-ch, Equals, item{typ: itemRightParen, val: ")"})
+	c.Check(<-ch, Equals, item{typ: itemAnd, val: ","})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "$Source"})
+	c.Check(<-ch, Equals, item{typ: itemOr, val: "|"})
+	c.Check(<-ch, Equals, item{typ: itemNot, val: "!"})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "app"})
+	c.Check(<-ch, Equals, item{typ: itemAnd, val: ","})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "d\"a'ta"})
+	c.Check(<-ch, Equals, item{typ: itemLeftCurly, val: "{"})
+	c.Check(<-ch, Equals, item{typ: itemString, val: "i386"})
+	c.Check(<-ch, Equals, item{typ: itemRightCurly, val: "}"})
+	c.Check(<-ch, Equals, item{typ: itemEOF, val: ""})
+}
+
+func (s *LexerSuite) TestConsume(c *C) {
+	l, _ := lex("query", "package (<< 1.3)")
+
+	c.Check(l.Current(), Equals, item{typ: itemString, val: "package"})
+	c.Check(l.Current(), Equals, item{typ: itemString, val: "package"})
+	l.Consume()
+	c.Check(l.Current(), Equals, item{typ: itemLeftParen, val: "("})
+	l.Consume()
+	c.Check(l.Current(), Equals, item{typ: itemLt, val: "<<"})
+}
+
+func (s *LexerSuite) TestString(c *C) {
+	l, _ := lex("query", "package (<< 1.3)")
+
+	c.Check(fmt.Sprintf("%s", l.Current()), Equals, "\"package\"")
+	l.Consume()
+	c.Check(fmt.Sprintf("%s", l.Current()), Equals, "(")
+}
+
+func (s *LexerSuite) TestError(c *C) {
+	l, _ := lex("query", "'package")
+
+	c.Check(fmt.Sprintf("%s", l.Current()), Equals, "error: unexpected eof in quoted string")
+}

query/query.go

@@ -0,0 +1,29 @@
+// Package query implements query language for
+package query
+
+import (
+	"github.com/smira/aptly/deb"
+)
+
+/*
+
+  Query language resembling Debian dependencies and reprepro
+  queries: http://mirrorer.alioth.debian.org/reprepro.1.html
+
+  Query := A | A '|' Query
+  A := B | B ',' A
+  B := C | '!' B
+  C := '(' Query ')' | D
+  D := <field> <condition> <arch_condition> | <pkg>_<version>_<arch>
+  field := <package-name> | <field> | $special_field
+  condition := '(' <operator> value ')' |
+  arch_condition := '{' arch '}' |
+  operator := | << | < | <= | > | >> | >= | = | % | ~
+*/
+
+// Parse parses input package query into PackageQuery tree ready for evaluation
+func Parse(query string) (result deb.PackageQuery, err error) {
+	l, _ := lex("", query)
+	result, err = parse(l)
+	return
+}

query/query_test.go

@@ -0,0 +1,11 @@
+package query
+
+import (
+	. "launchpad.net/gocheck"
+	"testing"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}