Disable support for aptly task for now. #96

Config option should act as default, while flag should override it only if set.

.travis.yml

@@ -1,15 +1,17 @@
 language: go
 
 go:
-  - 1.1
-  - 1.2
   - 1.2.1
+  - 1.3.1
   - tip
 
 env:
   global:
     - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
-
+    - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -y python-boto
 install:
   - make prepare
 

AUTHORS

@@ -0,0 +1,7 @@
+List of contributors, in chronological order:
+
+* Andrey Smirnov (https://github.com/smira)
+* Sebastien Binet (https://github.com/sbinet)
+* Ryan Uber (https://github.com/ryanuber)
+* Simon Aquino (https://github.com/simonaquino)
+* Vincent Batoufflet (https://github.com/vbatoufflet)

Gomfile

@@ -1,12 +1,17 @@
 gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
 gom 'code.google.com/p/go.crypto/ssh/terminal', :commit => '7aa593ce8cea'
-gom 'code.google.com/p/gographviz', :commit => '212766062629'
+gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2'
+gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556'
 gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
 gom 'github.com/cheggaaa/pb', :commit => '74be7a1388046f374ac36e93d46f5d56e856f827'
-gom 'github.com/gonuts/commander', :commit => 'f8ba4e959ca914268227c3ebbd7f6bf0bb35541a'
-gom 'github.com/gonuts/flag', :commit => '741a6cbd37a30dedc93f817e7de6aaf0ca38a493'
+gom 'github.com/vaughan0/go-ini', :commit => 'a98ad7ee00ec53921f08832bc06ecf7fd600e6a1'
+gom 'github.com/mattn/go-shellwords', :commit => 'c7ca6f94add751566a61cf2199e1de78d4c3eee4'
+gom 'github.com/mitchellh/goamz/s3', :commit => 'e7664b32019f31fd1bdf33f9e85f28722f700405'
 gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => '527a7b286bd095794af6c519627b7ed3d8fd067a'
+gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
+gom 'github.com/smira/flag', :commit => '357ed3e599ffcbd4aeaa828e1d10da2df3ea5107'
+gom 'github.com/smira/go-ftp-protocol/protocol', :commit => '066b75c2b70dca7ae10b1b88b47534a3c31ccfaa'
+gom 'github.com/syndtr/goleveldb/leveldb', :commit => 'e2fa4e6ac1cc41a73bc9fd467878ecbf65df5cc3'
 gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
 gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
 

Makefile

@@ -1,6 +1,6 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-PACKAGES=database debian files http utils
-ALL_PACKAGES=aptly cmd console database debian files http utils
+PACKAGES=database deb files http query s3 utils
+ALL_PACKAGES=aptly cmd console database deb files http query s3 utils
 BINPATH=$(abspath ./_vendor/bin)
 GOM_ENVIRONMENT=-test
 PYTHON?=python
@@ -43,9 +43,7 @@ install:
 	$(GOM) build -o $(BINPATH)/aptly
 
 system-test: install
-ifeq ($(GOVERSION),$(filter $(GOVERSION),go1.2 go1.2.1 devel))
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
-endif
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
 	PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long
 
@@ -61,4 +59,26 @@ mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
+package:
+	rm -rf root/
+	mkdir -p root/usr/bin/ root/usr/share/man/man1/ root/etc/bash_completion.d
+	cp $(BINPATH)/aptly root/usr/bin
+	cp man/aptly.1 root/usr/share/man/man1
+	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
+	gzip root/usr/share/man/man1/aptly.1
+	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
+	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 -C root/ .
+	mv aptly_$(VERSION)_*.deb ~
+
+src-package:
+	rm -rf aptly-$(VERSION)
+	mkdir -p aptly-$(VERSION)/src/github.com/smira/aptly/
+	cd aptly-$(VERSION)/src/github.com/smira/ && git clone https://github.com/smira/aptly && cd aptly && git checkout v$(VERSION)
+	cd aptly-$(VERSION)/src/github.com/smira/aptly && gom -production install
+	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . \( -name .git -o -name .bzr -o -name .hg \) -print | xargs rm -rf
+	rm -rf aptly-$(VERSION)/src/github.com/smira/aptly/_vendor/{pkg,bin}
+	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
+	rm -rf aptly-$(VERSION)
+	curl -T aptly-$(VERSION)-src.tar.bz2 -usmira:$(BINTRAY_KEY) https://api.bintray.com/content/smira/aptly/aptly/$(VERSION)/$(VERSION)/aptly-$(VERSION)-src.tar.bz2
+
 .PHONY: coverage.out

README.rst

@@ -8,8 +8,14 @@ aptly
 .. image:: https://coveralls.io/repos/smira/aptly/badge.png?branch=HEAD
     :target: https://coveralls.io/r/smira/aptly?branch=HEAD
 
+.. image:: http://gobuild.io/badge/github.com/smira/aptly/download.png
+    :target: http://gobuild.io/github.com/smira/aptly
+
 Aptly is a swiss army knife for Debian repository management.
 
+.. image:: http://www.aptly.info/img/aptly_logo.png
+    :target: http://www.aptly.info/
+
 Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support use
 mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
 
@@ -20,27 +26,44 @@ Aptly features: ("+" means planned features)
 * publish snapshot as Debian repository, ready to be consumed by apt
 * controlled update of one or more packages in snapshot from upstream mirror, tracking dependencies
 * merge two or more snapshots into one
-* filter repository by search query, pulling dependencies when required (+)
-* publish self-made packages as Debian repositories (+)
+* filter repository by search query, pulling dependencies when required
+* publish self-made packages as Debian repositories
 * mirror repositories "as-is" (without resigning with user's key) (+)
 * support for yum repositories (+)
 
 Current limitations:
 
-* debian-installer and translations not supported yet
+* translations are not supported yet
 
 Download
 --------
 
+To install aptly on Debian/Ubuntu, add new repository to /etc/apt/sources.list::
+
+    deb http://repo.aptly.info/ squeeze main
+
+And import key that is used to sign the release::
+
+    $ gpg --keyserver keys.gnupg.net --recv-keys 2A194991
+    $ gpg -a --export 2A194991 | sudo apt-key add -
+
+After that you can install aptly as any other software package::
+
+    $ apt-get update
+    $ apt-get install aptly
+
+Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
+Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
+
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.1+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.2+ required)::
 
     go get -u github.com/mattn/gom
     mkdir -p $GOPATH/src/github.com/smira/aptly
     git clone https://github.com/smira/aptly $GOPATH/src/github.com/smira/aptly
     cd $GOPATH/src/github.com/smira/aptly
-    gom install
+    gom -production install
     gom build -o $GOPATH/bin/aptly
 
 Aptly is using `gom <https://github.com/mattn/gom>`_ to fix external dependencies, so regular ``go get github.com/smira/aptly``

aptly/interfaces.go

@@ -5,7 +5,6 @@ package aptly
 import (
 	"github.com/smira/aptly/utils"
 	"io"
-	"os"
 )
 
 // PackagePool is asbtraction of package pool storage.
@@ -26,18 +25,32 @@ type PackagePool interface {
 
 // PublishedStorage is abstraction of filesystem storing all published repositories
 type PublishedStorage interface {
-	// PublicPath returns root of public part
-	PublicPath() string
 	// MkDir creates directory recursively under public path
 	MkDir(path string) error
-	// CreateFile creates file for writing under public path
-	CreateFile(path string) (*os.File, error)
+	// PutFile puts file into published storage at specified path
+	PutFile(path string, sourceFilename string) error
 	// RemoveDirs removes directory structure under public path
-	RemoveDirs(path string) error
+	RemoveDirs(path string, progress Progress) error
+	// Remove removes single file under public path
+	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(prefix string, component string, poolDirectory string, sourcePool PackagePool, sourcePath string) (string, error)
-	// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
-	ChecksumsForFile(path string) (utils.ChecksumInfo, error)
+	LinkFromPool(publishedDirectory string, sourcePool PackagePool, sourcePath, sourceMD5 string, force bool) error
+	// Filelist returns list of files under prefix
+	Filelist(prefix string) ([]string, error)
+	// RenameFile renames (moves) file
+	RenameFile(oldName, newName string) error
+}
+
+// LocalPublishedStorage is published storage on local filesystem
+type LocalPublishedStorage interface {
+	// PublicPath returns root of public part
+	PublicPath() string
+}
+
+// PublishedStorageProvider is a thing that returns PublishedStorage by name
+type PublishedStorageProvider interface {
+	// GetPublishedStorage returns PublishedStorage by name
+	GetPublishedStorage(name string) PublishedStorage
 }
 
 // Progress is a progress displaying entity, it allows progress bars & simple prints
@@ -77,6 +90,8 @@ type Downloader interface {
 	// Shutdown stops downloader after current tasks are finished,
 	// but doesn't process rest of queue
 	Shutdown()
+	// Abort stops downloader without waiting for shutdown
+	Abort()
 	// GetProgress returns Progress object
 	GetProgress() Progress
 }

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
 // Version of aptly
-const Version = "0.4.1"
+const Version = "0.8"
 
 // Enable debugging features?
 const EnableDebug = false

cmd/cmd.go

@@ -3,28 +3,26 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
 	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"os"
 	"time"
 )
 
 // ListPackagesRefList shows list of packages in PackageRefList
-func ListPackagesRefList(reflist *debian.PackageRefList) (err error) {
+func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 	fmt.Printf("Packages:\n")
 
 	if reflist == nil {
 		return
 	}
 
-	packageCollection := debian.NewPackageCollection(context.database)
-
 	err = reflist.ForEach(func(key []byte) error {
-		p, err := packageCollection.ByKey(key)
-		if err != nil {
-			return err
+		p, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
+		if err2 != nil {
+			return err2
 		}
 		fmt.Printf("  %s\n", p)
 		return nil
@@ -36,6 +34,18 @@ func ListPackagesRefList(reflist *debian.PackageRefList) (err error) {
 	return
 }
 
+// LookupOption checks boolean flag with default (usually config) and command-line
+// setting
+func LookupOption(defaultValue bool, flags *flag.FlagSet, name string) (result bool) {
+	result = defaultValue
+
+	if flags.IsSet(name) {
+		result = flags.Lookup(name).Value.Get().(bool)
+	}
+
+	return
+}
+
 // RootCommand creates root command in command tree
 func RootCommand() *commander.Command {
 	cmd := &commander.Command{
@@ -47,10 +57,10 @@ repositories, manage local repositories, filter them, merge,
 upgrade individual packages, take snapshots and publish them
 back as Debian repositories.
 
-aptly goal is to establish repeatiblity and controlled changes
-in package environment. aptly allows to fix set of packages in
-repository, so that package installation and upgrade becomes
-deterministic. At the same time aptly allows to perform controlled,
+aptly's goal is to establish repeatability and controlled changes
+in a package-centric environment. aptly allows one to fix a set of packages
+in a repository, so that package installation and upgrade becomes
+deterministic. At the same time aptly allows one to perform controlled,
 fine-grained changes in repository contents to transition your
 package environment to new version.`,
 		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
@@ -61,8 +71,11 @@ package environment to new version.`,
 			makeCmdRepo(),
 			makeCmdServe(),
 			makeCmdSnapshot(),
+			// Disabled on no docs
+			//makeCmdTask(),
 			makeCmdPublish(),
 			makeCmdVersion(),
+			makeCmdPackage(),
 		},
 	}
 

cmd/context.go

@@ -2,14 +2,16 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/console"
 	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/s3"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -18,13 +20,17 @@ import (
 	"time"
 )
 
-// Common context shared by all commands
-var context struct {
+// AptlyContext is a common context shared by all commands
+type AptlyContext struct {
+	flags, globalFlags *flag.FlagSet
+	configLoaded       bool
+
 	progress          aptly.Progress
 	downloader        aptly.Downloader
 	database          database.Storage
 	packagePool       aptly.PackagePool
-	publishedStorage  aptly.PublishedStorage
+	publishedStorages map[string]aptly.PublishedStorage
+	collectionFactory *deb.CollectionFactory
 	dependencyOptions int
 	architecturesList []string
 	// Debug features
@@ -33,45 +39,294 @@ var context struct {
 	fileMemStats   *os.File
 }
 
+var context *AptlyContext
+
+// Check interface
+var _ aptly.PublishedStorageProvider = &AptlyContext{}
+
+// FatalError is type for panicking to abort execution with non-zero
+// exit code and print meaningful explanation
+type FatalError struct {
+	ReturnCode int
+	Message    string
+}
+
+// Fatal panics and aborts execution with exit code 1
+func Fatal(err error) {
+	returnCode := 1
+	if err == commander.ErrFlagError || err == commander.ErrCommandError {
+		returnCode = 2
+	}
+	panic(&FatalError{ReturnCode: returnCode, Message: err.Error()})
+}
+
+// Config loads and returns current configuration
+func (context *AptlyContext) Config() *utils.ConfigStructure {
+	if !context.configLoaded {
+		var err error
+
+		configLocation := context.globalFlags.Lookup("config").Value.String()
+		if configLocation != "" {
+			err = utils.LoadConfig(configLocation, &utils.Config)
+
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			configLocations := []string{
+				filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
+				"/etc/aptly.conf",
+			}
+
+			for _, configLocation := range configLocations {
+				err = utils.LoadConfig(configLocation, &utils.Config)
+				if err == nil {
+					break
+				}
+				if !os.IsNotExist(err) {
+					Fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
+				}
+			}
+
+			if err != nil {
+				fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
+				utils.SaveConfig(configLocations[0], &utils.Config)
+			}
+		}
+
+		context.configLoaded = true
+
+	}
+	return &utils.Config
+}
+
+// DependencyOptions calculates options related to dependecy handling
+func (context *AptlyContext) DependencyOptions() int {
+	if context.dependencyOptions == -1 {
+		context.dependencyOptions = 0
+		if LookupOption(context.Config().DepFollowSuggests, context.globalFlags, "dep-follow-suggests") {
+			context.dependencyOptions |= deb.DepFollowSuggests
+		}
+		if LookupOption(context.Config().DepFollowRecommends, context.globalFlags, "dep-follow-recommends") {
+			context.dependencyOptions |= deb.DepFollowRecommends
+		}
+		if LookupOption(context.Config().DepFollowAllVariants, context.globalFlags, "dep-follow-all-variants") {
+			context.dependencyOptions |= deb.DepFollowAllVariants
+		}
+		if LookupOption(context.Config().DepFollowSource, context.globalFlags, "dep-follow-source") {
+			context.dependencyOptions |= deb.DepFollowSource
+		}
+	}
+
+	return context.dependencyOptions
+}
+
+// ArchitecturesList returns list of architectures fixed via command line or config
+func (context *AptlyContext) ArchitecturesList() []string {
+	if context.architecturesList == nil {
+		context.architecturesList = context.Config().Architectures
+		optionArchitectures := context.globalFlags.Lookup("architectures").Value.String()
+		if optionArchitectures != "" {
+			context.architecturesList = strings.Split(optionArchitectures, ",")
+		}
+	}
+
+	return context.architecturesList
+}
+
+// Progress creates or returns Progress object
+func (context *AptlyContext) Progress() aptly.Progress {
+	if context.progress == nil {
+		context.progress = console.NewProgress()
+		context.progress.Start()
+	}
+
+	return context.progress
+}
+
+// Downloader returns instance of current downloader
+func (context *AptlyContext) Downloader() aptly.Downloader {
+	if context.downloader == nil {
+		var downloadLimit int64
+		limitFlag := context.flags.Lookup("download-limit")
+		if limitFlag != nil {
+			downloadLimit = limitFlag.Value.Get().(int64)
+		}
+		if downloadLimit == 0 {
+			downloadLimit = context.Config().DownloadLimit
+		}
+		context.downloader = http.NewDownloader(context.Config().DownloadConcurrency,
+			downloadLimit*1024, context.Progress())
+	}
+
+	return context.downloader
+}
+
+// DBPath builds path to database
+func (context *AptlyContext) DBPath() string {
+	return filepath.Join(context.Config().RootDir, "db")
+}
+
+// Database opens and returns current instance of database
+func (context *AptlyContext) Database() (database.Storage, error) {
+	if context.database == nil {
+		var err error
+
+		context.database, err = database.OpenDB(context.DBPath())
+		if err != nil {
+			return nil, fmt.Errorf("can't open database: %s", err)
+		}
+	}
+
+	return context.database, nil
+}
+
+// CloseDatabase closes the db temporarily
+func (context *AptlyContext) CloseDatabase() error {
+	if context.database == nil {
+		return nil
+	}
+
+	return context.database.Close()
+}
+
+// ReOpenDatabase reopens the db after close
+func (context *AptlyContext) ReOpenDatabase() error {
+	if context.database == nil {
+		return nil
+	}
+
+	const MaxTries = 10
+	const Delay = 10 * time.Second
+
+	for try := 0; try < MaxTries; try++ {
+		err := context.database.ReOpen()
+		if err == nil || strings.Index(err.Error(), "resource temporarily unavailable") == -1 {
+			return err
+		}
+		context.Progress().Printf("Unable to reopen database, sleeping %s\n", Delay)
+		<-time.After(Delay)
+	}
+
+	return fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
+}
+
+// CollectionFactory builds factory producing all kinds of collections
+func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
+	if context.collectionFactory == nil {
+		db, err := context.Database()
+		if err != nil {
+			Fatal(err)
+		}
+		context.collectionFactory = deb.NewCollectionFactory(db)
+	}
+
+	return context.collectionFactory
+}
+
+// PackagePool returns instance of PackagePool
+func (context *AptlyContext) PackagePool() aptly.PackagePool {
+	if context.packagePool == nil {
+		context.packagePool = files.NewPackagePool(context.Config().RootDir)
+	}
+
+	return context.packagePool
+}
+
+// GetPublishedStorage returns instance of PublishedStorage
+func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
+	publishedStorage, ok := context.publishedStorages[name]
+	if !ok {
+		if name == "" {
+			publishedStorage = files.NewPublishedStorage(context.Config().RootDir)
+		} else if strings.HasPrefix(name, "s3:") {
+			params, ok := context.Config().S3PublishRoots[name[3:]]
+			if !ok {
+				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
+			}
+
+			var err error
+			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
+				params.Region, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
+				params.EncryptionMethod, params.PlusWorkaround)
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			Fatal(fmt.Errorf("unknown published storage format: %v", name))
+		}
+		context.publishedStorages[name] = publishedStorage
+	}
+
+	return publishedStorage
+}
+
+// UpdateFlags sets internal copy of flags in the context
+func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) {
+	context.flags = flags
+}
+
+// ShutdownContext shuts context down
+func ShutdownContext() {
+	if aptly.EnableDebug {
+		if context.fileMemProfile != nil {
+			pprof.WriteHeapProfile(context.fileMemProfile)
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+		if context.fileCPUProfile != nil {
+			pprof.StopCPUProfile()
+			context.fileCPUProfile.Close()
+			context.fileCPUProfile = nil
+		}
+		if context.fileMemProfile != nil {
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+	}
+	if context.database != nil {
+		context.database.Close()
+		context.database = nil
+	}
+	if context.downloader != nil {
+		context.downloader.Abort()
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
+// CleanupContext does partial shutdown of context
+func CleanupContext() {
+	if context.downloader != nil {
+		context.downloader.Shutdown()
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
 // InitContext initializes context with default settings
-func InitContext(cmd *commander.Command) error {
+func InitContext(flags *flag.FlagSet) error {
 	var err error
 
-	context.dependencyOptions = 0
-	if utils.Config.DepFollowSuggests || cmd.Flag.Lookup("dep-follow-suggests").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowSuggests
-	}
-	if utils.Config.DepFollowRecommends || cmd.Flag.Lookup("dep-follow-recommends").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowRecommends
-	}
-	if utils.Config.DepFollowAllVariants || cmd.Flag.Lookup("dep-follow-all-variants").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowAllVariants
-	}
-	if utils.Config.DepFollowSource || cmd.Flag.Lookup("dep-follow-source").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowSource
+	if context != nil {
+		panic("context already initialized")
 	}
 
-	context.architecturesList = utils.Config.Architectures
-	optionArchitectures := cmd.Flag.Lookup("architectures").Value.String()
-	if optionArchitectures != "" {
-		context.architecturesList = strings.Split(optionArchitectures, ",")
+	context = &AptlyContext{
+		flags:             flags,
+		globalFlags:       flags,
+		dependencyOptions: -1,
+		publishedStorages: map[string]aptly.PublishedStorage{},
 	}
 
-	context.progress = console.NewProgress()
-	context.progress.Start()
-
-	context.downloader = http.NewDownloader(utils.Config.DownloadConcurrency, context.progress)
-
-	context.database, err = database.OpenDB(filepath.Join(utils.Config.RootDir, "db"))
-	if err != nil {
-		return fmt.Errorf("can't open database: %s", err)
-	}
-
-	context.packagePool = files.NewPackagePool(utils.Config.RootDir)
-	context.publishedStorage = files.NewPublishedStorage(utils.Config.RootDir)
-
 	if aptly.EnableDebug {
-		cpuprofile := cmd.Flag.Lookup("cpuprofile").Value.String()
+		cpuprofile := flags.Lookup("cpuprofile").Value.String()
 		if cpuprofile != "" {
 			context.fileCPUProfile, err = os.Create(cpuprofile)
 			if err != nil {
@@ -80,7 +335,7 @@ func InitContext(cmd *commander.Command) error {
 			pprof.StartCPUProfile(context.fileCPUProfile)
 		}
 
-		memprofile := cmd.Flag.Lookup("memprofile").Value.String()
+		memprofile := flags.Lookup("memprofile").Value.String()
 		if memprofile != "" {
 			context.fileMemProfile, err = os.Create(memprofile)
 			if err != nil {
@@ -88,9 +343,9 @@ func InitContext(cmd *commander.Command) error {
 			}
 		}
 
-		memstats := cmd.Flag.Lookup("memstats").Value.String()
+		memstats := flags.Lookup("memstats").Value.String()
 		if memstats != "" {
-			interval := cmd.Flag.Lookup("meminterval").Value.Get().(time.Duration)
+			interval := flags.Lookup("meminterval").Value.Get().(time.Duration)
 
 			context.fileMemStats, err = os.Create(memstats)
 			if err != nil {
@@ -120,32 +375,3 @@ func InitContext(cmd *commander.Command) error {
 
 	return nil
 }
-
-// ShutdownContext shuts context down
-func ShutdownContext() {
-	if aptly.EnableDebug {
-		if context.fileMemProfile != nil {
-			pprof.WriteHeapProfile(context.fileMemProfile)
-			context.fileMemProfile.Close()
-			context.fileMemProfile = nil
-		}
-		if context.fileCPUProfile != nil {
-			pprof.StopCPUProfile()
-			context.fileCPUProfile.Close()
-			context.fileCPUProfile = nil
-		}
-		if context.fileMemProfile != nil {
-			context.fileMemProfile.Close()
-			context.fileMemProfile = nil
-		}
-	}
-	if context.database != nil {
-		context.database.Close()
-	}
-	if context.downloader != nil {
-		context.downloader.Shutdown()
-	}
-	if context.progress != nil {
-		context.progress.Shutdown()
-	}
-}

cmd/db.go

@@ -1,8 +1,7 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
 )
 
 func makeCmdDb() *commander.Command {
@@ -11,7 +10,7 @@ func makeCmdDb() *commander.Command {
 		Short:     "manage aptly's internal database and package pool",
 		Subcommands: []*commander.Command{
 			makeCmdDbCleanup(),
+			makeCmdDbRecover(),
 		},
-		Flag: *flag.NewFlagSet("aptly-db", flag.ExitOnError),
 	}
 }

cmd/db_cleanup.go

@@ -2,10 +2,9 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
 	"sort"
 )
 
@@ -15,16 +14,15 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	// collect information about references packages...
-	existingPackageRefs := debian.NewPackageRefList()
+	existingPackageRefs := deb.NewPackageRefList()
 
-	context.progress.Printf("Loading mirrors, local repos and snapshots...\n")
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-	err = repoCollection.ForEach(func(repo *debian.RemoteRepo) error {
-		err := repoCollection.LoadComplete(repo)
+	context.Progress().Printf("Loading mirrors, local repos and snapshots...\n")
+	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
@@ -37,9 +35,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-	err = localRepoCollection.ForEach(func(repo *debian.LocalRepo) error {
-		err := localRepoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
@@ -52,9 +49,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	err = snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
-		err := snapshotCollection.LoadComplete(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return err
 		}
@@ -66,44 +62,45 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	}
 
 	// ... and compare it to the list of all packages
-	context.progress.Printf("Loading list of all packages...\n")
-	packageCollection := debian.NewPackageCollection(context.database)
-	allPackageRefs := packageCollection.AllPackageRefs()
+	context.Progress().Printf("Loading list of all packages...\n")
+	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
 
 	toDelete := allPackageRefs.Substract(existingPackageRefs)
 
 	// delete packages that are no longer referenced
-	context.progress.Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
+	context.Progress().Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
 
-	context.database.StartBatch()
+	// database can't err as collection factory already constructed
+	db, _ := context.Database()
+	db.StartBatch()
 	err = toDelete.ForEach(func(ref []byte) error {
-		return packageCollection.DeleteByKey(ref)
+		return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
 	})
 	if err != nil {
 		return err
 	}
 
-	err = context.database.FinishBatch()
+	err = db.FinishBatch()
 	if err != nil {
 		return fmt.Errorf("unable to write to DB: %s", err)
 	}
 
 	// now, build a list of files that should be present in Repository (package pool)
-	context.progress.Printf("Building list of files referenced by packages...\n")
+	context.Progress().Printf("Building list of files referenced by packages...\n")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())
-	context.progress.InitBar(int64(existingPackageRefs.Len()), false)
+	context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
 
 	err = existingPackageRefs.ForEach(func(key []byte) error {
-		pkg, err := packageCollection.ByKey(key)
-		if err != nil {
-			return err
+		pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
+		if err2 != nil {
+			return err2
 		}
-		paths, err := pkg.FilepathList(context.packagePool)
-		if err != nil {
-			return err
+		paths, err2 := pkg.FilepathList(context.PackagePool())
+		if err2 != nil {
+			return err2
 		}
 		referencedFiles = append(referencedFiles, paths...)
-		context.progress.AddBar(1)
+		context.Progress().AddBar(1)
 
 		return nil
 	})
@@ -112,11 +109,11 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	}
 
 	sort.Strings(referencedFiles)
-	context.progress.ShutdownBar()
+	context.Progress().ShutdownBar()
 
 	// build a list of files in the package pool
-	context.progress.Printf("Building list of files in package pool...\n")
-	existingFiles, err := context.packagePool.FilepathList(context.progress)
+	context.Progress().Printf("Building list of files in package pool...\n")
+	existingFiles, err := context.PackagePool().FilepathList(context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to collect file paths: %s", err)
 	}
@@ -125,24 +122,29 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
 
 	// delete files that are no longer referenced
-	context.progress.Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
+	context.Progress().Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
 
 	if len(filesToDelete) > 0 {
-		context.progress.InitBar(int64(len(filesToDelete)), false)
-		totalSize := int64(0)
+		context.Progress().InitBar(int64(len(filesToDelete)), false)
+
+		var size, totalSize int64
 		for _, file := range filesToDelete {
-			size, err := context.packagePool.Remove(file)
+			size, err = context.PackagePool().Remove(file)
 			if err != nil {
 				return err
 			}
 
-			context.progress.AddBar(1)
+			context.Progress().AddBar(1)
 			totalSize += size
 		}
-		context.progress.ShutdownBar()
+		context.Progress().ShutdownBar()
 
-		context.progress.Printf("Disk space freed: %.2f GiB...\n", float64(totalSize)/1024.0/1024.0/1024.0)
+		context.Progress().Printf("Disk space freed: %s...\n", utils.HumanBytes(totalSize))
 	}
+
+	context.Progress().Printf("Compacting database...\n")
+	err = db.CompactDB()
+
 	return err
 }
 
@@ -159,7 +161,6 @@ Example:
 
   $ aptly db cleanup
 `,
-		Flag: *flag.NewFlagSet("aptly-db-cleanup", flag.ExitOnError),
 	}
 
 	return cmd

cmd/db_recover.go

@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"github.com/smira/aptly/database"
+	"github.com/smira/commander"
+)
+
+// aptly db recover
+func aptlyDbRecover(cmd *commander.Command, args []string) error {
+	var err error
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	context.Progress().Printf("Recovering database...\n")
+	err = database.RecoverDB(context.DBPath())
+
+	return err
+}
+
+func makeCmdDbRecover() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyDbRecover,
+		UsageLine: "recover",
+		Short:     "recover DB after crash",
+		Long: `
+Database recover does its' best to recover the database after a crash.
+It is recommended to backup the DB before running recover.
+
+Example:
+
+  $ aptly db recover
+`,
+	}
+
+	return cmd
+}

cmd/graph.go

@@ -4,9 +4,8 @@ import (
 	"bytes"
 	"code.google.com/p/gographviz"
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"io"
 	"io/ioutil"
 	"os"
@@ -14,14 +13,15 @@ import (
 	"strings"
 )
 
-func graphvizEscape(s string) string {
-	return fmt.Sprintf("\"%s\"", strings.Replace(s, "\"", "\\\"", 0))
-}
-
 func aptlyGraph(cmd *commander.Command, args []string) error {
 	var err error
 
-	graph := gographviz.NewGraph()
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	graph := gographviz.NewEscape()
 	graph.SetDir(true)
 	graph.SetName("aptly")
 
@@ -29,21 +29,19 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Loading mirrors...\n")
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-
-	err = repoCollection.ForEach(func(repo *debian.RemoteRepo) error {
-		err := repoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
 
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
+		graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "darkgoldenrod1",
-			"label": graphvizEscape(fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
+			"label": fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
 				repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "),
-				strings.Join(repo.Architectures, ", "), repo.NumPackages())),
+				strings.Join(repo.Architectures, ", "), repo.NumPackages()),
 		})
 		existingNodes[repo.UUID] = true
 		return nil
@@ -55,20 +53,18 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Loading local repos...\n")
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-
-	err = localRepoCollection.ForEach(func(repo *debian.LocalRepo) error {
-		err := localRepoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
 
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
+		graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "mediumseagreen",
-			"label": graphvizEscape(fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
-				repo.Name, repo.Comment, repo.NumPackages())),
+			"label": fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
+				repo.Name, repo.Comment, repo.NumPackages()),
 		})
 		existingNodes[repo.UUID] = true
 		return nil
@@ -80,15 +76,13 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Loading snapshots...\n")
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-
-	snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
+	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
 		existingNodes[snapshot.UUID] = true
 		return nil
 	})
 
-	err = snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
-		err := snapshotCollection.LoadComplete(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return err
 		}
@@ -98,18 +92,18 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 			description = "Snapshot from repo"
 		}
 
-		graph.AddNode("aptly", graphvizEscape(snapshot.UUID), map[string]string{
+		graph.AddNode("aptly", snapshot.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "cadetblue1",
-			"label":     graphvizEscape(fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages())),
+			"label":     fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages()),
 		})
 
 		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
 			for _, uuid := range snapshot.SourceIDs {
 				_, exists := existingNodes[uuid]
 				if exists {
-					graph.AddEdge(graphvizEscape(uuid), "", graphvizEscape(snapshot.UUID), "", true, nil)
+					graph.AddEdge(uuid, snapshot.UUID, true, nil)
 				}
 			}
 		}
@@ -122,19 +116,20 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Loading published repos...\n")
 
-	publishedCollection := debian.NewPublishedRepoCollection(context.database)
-
-	publishedCollection.ForEach(func(repo *debian.PublishedRepo) error {
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
+	context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "darkolivegreen1",
-			"label":     graphvizEscape(fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution, repo.Component, strings.Join(repo.Architectures, ", "))),
+			"label": fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution,
+				strings.Join(repo.Components(), " "), strings.Join(repo.Architectures, ", ")),
 		})
 
-		_, exists := existingNodes[repo.SnapshotUUID]
-		if exists {
-			graph.AddEdge(graphvizEscape(repo.SnapshotUUID), "", graphvizEscape(repo.UUID), "", true, nil)
+		for _, uuid := range repo.Sources {
+			_, exists := existingNodes[uuid]
+			if exists {
+				graph.AddEdge(uuid, repo.UUID, true, nil)
+			}
 		}
 
 		return nil
@@ -198,13 +193,12 @@ func makeCmdGraph() *commander.Command {
 		Long: `
 Command graph displays relationship between mirrors, local repositories,
 snapshots and published repositories using graphviz package to render
-graph as image.
+graph as an image.
 
 Example:
 
   $ aptly graph
 `,
-		Flag: *flag.NewFlagSet("aptly-graph", flag.ExitOnError),
 	}
 
 	return cmd

cmd/mirror.go

@@ -1,19 +1,21 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"strings"
 )
 
-func getVerifier(cmd *commander.Command) (utils.Verifier, error) {
-	if utils.Config.GpgDisableVerify || cmd.Flag.Lookup("ignore-signatures").Value.Get().(bool) {
+func getVerifier(flags *flag.FlagSet) (utils.Verifier, error) {
+	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
 		return nil, nil
 	}
 
+	keyRings := flags.Lookup("keyring").Value.Get().([]string)
+
 	verifier := &utils.GpgVerifier{}
-	for _, keyRing := range keyRings.keyRings {
+	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}
 
@@ -42,8 +44,6 @@ func (k *keyRingsFlag) String() string {
 	return strings.Join(k.keyRings, ",")
 }
 
-var keyRings = keyRingsFlag{}
-
 func makeCmdMirror() *commander.Command {
 	return &commander.Command{
 		UsageLine: "mirror",
@@ -54,7 +54,9 @@ func makeCmdMirror() *commander.Command {
 			makeCmdMirrorShow(),
 			makeCmdMirrorDrop(),
 			makeCmdMirrorUpdate(),
+			makeCmdMirrorRename(),
+			makeCmdMirrorEdit(),
+			makeCmdMirrorSearch(),
 		},
-		Flag: *flag.NewFlagSet("aptly-mirror", flag.ExitOnError),
 	}
 }

cmd/mirror_create.go

@@ -2,10 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"strings"
 )
 
@@ -13,10 +13,11 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	var err error
 	if !(len(args) == 2 && strings.HasPrefix(args[1], "ppa:") || len(args) >= 3) {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	downloadSources := utils.Config.DownloadSourcePackages || cmd.Flag.Lookup("with-sources").Value.Get().(bool)
+	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.flags, "with-sources")
+	downloadUdebs := context.flags.Lookup("with-udebs").Value.Get().(bool)
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -25,7 +26,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 
 	mirrorName = args[0]
 	if len(args) == 2 {
-		archiveURL, distribution, components, err = debian.ParsePPA(args[1])
+		archiveURL, distribution, components, err = deb.ParsePPA(args[1], context.Config())
 		if err != nil {
 			return err
 		}
@@ -33,24 +34,33 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		archiveURL, distribution, components = args[1], args[2], args[3:]
 	}
 
-	repo, err := debian.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.architecturesList, downloadSources)
+	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
+		downloadSources, downloadUdebs)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
-	verifier, err := getVerifier(cmd)
+	repo.Filter = context.flags.Lookup("filter").Value.String()
+	repo.FilterWithDeps = context.flags.Lookup("filter-with-deps").Value.Get().(bool)
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to create mirror: %s", err)
+		}
+	}
+
+	verifier, err := getVerifier(context.flags)
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.downloader, verifier)
+	err = repo.Fetch(context.Downloader(), verifier)
 	if err != nil {
 		return fmt.Errorf("unable to fetch mirror: %s", err)
 	}
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-
-	err = repoCollection.Add(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add mirror: %s", err)
 	}
@@ -66,7 +76,8 @@ func makeCmdMirrorCreate() *commander.Command {
 		Short:     "create new mirror",
 		Long: `
 Creates mirror <name> of remote repository, aptly supports both regular and flat Debian repositories exported
-via HTTP. aptly would try download Release file from remote repository and verify its signature.
+via HTTP and FTP. aptly would try download Release file from remote repository and verify its' signature. Command
+line format resembles apt utlitily sources.list(5).
 
 PPA urls could specified in short format:
 
@@ -81,7 +92,10 @@ Example:
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
-	cmd.Flag.Var(&keyRings, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd
 }

cmd/mirror_drop.go

@@ -2,30 +2,32 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-	repo, err := repoCollection.ByName(name)
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	force := cmd.Flag.Lookup("force").Value.Get().(bool)
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	force := context.flags.Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshotCollection := debian.NewSnapshotCollection(context.database)
-		snapshots := snapshotCollection.ByRemoteRepoSource(repo)
+		snapshots := context.CollectionFactory().SnapshotCollection().ByRemoteRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Mirror `%s` was used to create following snapshots:\n", repo.Name)
@@ -37,7 +39,7 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = repoCollection.Drop(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
@@ -54,7 +56,7 @@ func makeCmdMirrorDrop() *commander.Command {
 		Short:     "delete mirror",
 		Long: `
 Drop deletes information about remote repository mirror <name>. Package data is not deleted
-(it could be still used by other mirrors or snapshots).  If mirror is used as source
+(since it could still be used by other mirrors or snapshots).  If mirror is used as source
 to create a snapshot, aptly would refuse to delete such mirror, use flag -force to override.
 
 Example:

cmd/mirror_edit.go

@@ -0,0 +1,91 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	context.flags.Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "filter":
+			repo.Filter = flag.Value.String()
+		case "filter-with-deps":
+			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-sources":
+			repo.DownloadSources = flag.Value.Get().(bool)
+		case "with-udebs":
+			repo.DownloadUdebs = flag.Value.Get().(bool)
+		}
+	})
+
+	if repo.IsFlat() && repo.DownloadUdebs {
+		return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
+	}
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
+	if context.globalFlags.Lookup("architectures").Value.String() != "" {
+		repo.Architectures = context.ArchitecturesList()
+
+		err = repo.Fetch(context.Downloader(), nil)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fmt.Printf("Mirror %s successfully updated.\n", repo)
+	return err
+}
+
+func makeCmdMirrorEdit() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorEdit,
+		UsageLine: "edit <name>",
+		Short:     "edit mirror settings",
+		Long: `
+Command edit allows one to change settings of mirror:
+filters, list of architectures.
+
+Example:
+
+  $ aptly mirror edit -filter=nginx -filter-with-deps some-mirror
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-edit", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+
+	return cmd
+}

cmd/mirror_list.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"sort"
 )
 
@@ -12,29 +11,40 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	if repoCollection.Len() > 0 {
-		fmt.Printf("List of mirrors:\n")
-		repos := make([]string, repoCollection.Len())
-		i := 0
-		repoCollection.ForEach(func(repo *debian.RemoteRepo) error {
+	repos := make([]string, context.CollectionFactory().RemoteRepoCollection().Len())
+	i := 0
+	context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		if raw {
+			repos[i] = repo.Name
+		} else {
 			repos[i] = repo.String()
-			i++
-			return nil
-		})
-
-		sort.Strings(repos)
-		for _, repo := range repos {
-			fmt.Printf(" * %s\n", repo)
 		}
+		i++
+		return nil
+	})
 
-		fmt.Printf("\nTo get more information about mirror, run `aptly mirror show <name>`.\n")
+	sort.Strings(repos)
+
+	if raw {
+		for _, repo := range repos {
+			fmt.Printf("%s\n", repo)
+		}
 	} else {
-		fmt.Printf("No mirrors found, create one with `aptly mirror create ...`.\n")
+		if len(repos) > 0 {
+			fmt.Printf("List of mirrors:\n")
+			for _, repo := range repos {
+				fmt.Printf(" * %s\n", repo)
+			}
+
+			fmt.Printf("\nTo get more information about mirror, run `aptly mirror show <name>`.\n")
+		} else {
+			fmt.Printf("No mirrors found, create one with `aptly mirror create ...`.\n")
+		}
 	}
 	return err
 }
@@ -51,8 +61,9 @@ Example:
 
   $ aptly mirror list
 `,
-		Flag: *flag.NewFlagSet("aptly-mirror-list", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+
 	return cmd
 }

cmd/mirror_rename.go

@@ -0,0 +1,64 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyMirrorRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.RemoteRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().RemoteRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: mirror %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nMirror %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdMirrorRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames mirror",
+		Long: `
+Command changes name of the mirror.Mirror name should be unique.
+
+Example:
+
+  $ aptly mirror rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/mirror_search.go

@@ -0,0 +1,26 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdMirrorSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> <package-query>",
+		Short:     "search mirror for packages matching query",
+		Long: `
+Command search displays list of packages in mirror that match package query
+
+Example:
+
+    $ aptly mirror search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+
+	return cmd
+}

cmd/mirror_show.go

@@ -2,10 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"strings"
 )
 
@@ -13,23 +13,25 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-	repo, err := repoCollection.ByName(name)
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = repoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
 	fmt.Printf("Name: %s\n", repo.Name)
+	if repo.Status == deb.MirrorUpdating {
+		fmt.Printf("Status: In Update (PID %d)\n", repo.WorkerPID)
+	}
 	fmt.Printf("Archive Root URL: %s\n", repo.ArchiveRoot)
 	fmt.Printf("Distribution: %s\n", repo.Distribution)
 	fmt.Printf("Components: %s\n", strings.Join(repo.Components, ", "))
@@ -39,6 +41,19 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		downloadSources = "yes"
 	}
 	fmt.Printf("Download Sources: %s\n", downloadSources)
+	downloadUdebs := "no"
+	if repo.DownloadUdebs {
+		downloadUdebs = "yes"
+	}
+	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
+	if repo.Filter != "" {
+		fmt.Printf("Filter: %s\n", repo.Filter)
+		filterWithDeps := "no"
+		if repo.FilterWithDeps {
+			filterWithDeps = "yes"
+		}
+		fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
+	}
 	if repo.LastDownloadDate.IsZero() {
 		fmt.Printf("Last update: never\n")
 	} else {
@@ -51,7 +66,7 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		fmt.Printf("%s: %s\n", k, repo.Meta[k])
 	}
 
-	withPackages := cmd.Flag.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		if repo.LastDownloadDate.IsZero() {
 			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
@@ -69,7 +84,7 @@ func makeCmdMirrorShow() *commander.Command {
 		UsageLine: "show <name>",
 		Short:     "show details about mirror",
 		Long: `
-Shows detailed information about mirror.
+Shows detailed information about the mirror.
 
 Example:
 

cmd/mirror_update.go

@@ -2,56 +2,167 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"os"
+	"os/signal"
+	"strings"
 )
 
 func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-	repo, err := repoCollection.ByName(name)
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = repoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	ignoreMismatch := cmd.Flag.Lookup("ignore-checksums").Value.Get().(bool)
+	force := context.flags.Lookup("force").Value.Get().(bool)
+	if !force {
+		err = repo.CheckLock()
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
 
-	verifier, err := getVerifier(cmd)
+	ignoreMismatch := context.flags.Lookup("ignore-checksums").Value.Get().(bool)
+
+	verifier, err := getVerifier(context.flags)
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.downloader, verifier)
+	err = repo.Fetch(context.Downloader(), verifier)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	packageCollection := debian.NewPackageCollection(context.database)
-
-	err = repo.Download(context.progress, context.downloader, packageCollection, context.packagePool, ignoreMismatch)
+	context.Progress().Printf("Downloading & parsing package files...\n")
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = repoCollection.Update(repo)
+	if repo.Filter != "" {
+		context.Progress().Printf("Applying filter...\n")
+		var filterQuery deb.PackageQuery
+
+		filterQuery, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+
+		var oldLen, newLen int
+		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+		context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen)
+	}
+
+	var (
+		downloadSize int64
+		queue        []deb.PackageDownloadTask
+	)
+
+	context.Progress().Printf("Building download queue...\n")
+	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool())
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	context.progress.Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
+	defer func() {
+		// on any interruption, unlock the mirror
+		err := context.ReOpenDatabase()
+		if err == nil {
+			repo.MarkAsIdle()
+			context.CollectionFactory().RemoteRepoCollection().Update(repo)
+		}
+	}()
+
+	repo.MarkAsUpdating()
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = context.CloseDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	// Catch ^C
+	sigch := make(chan os.Signal)
+	signal.Notify(sigch, os.Interrupt)
+
+	count := len(queue)
+	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
+
+	// Download from the queue
+	context.Progress().InitBar(downloadSize, true)
+
+	// Download all package files
+	ch := make(chan error, count)
+
+	// In separate goroutine (to avoid blocking main), push queue to downloader
+	go func() {
+		for _, task := range queue {
+			context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
+		}
+
+		// We don't need queue after this point
+		queue = nil
+	}()
+
+	// Wait for all downloads to finish
+	errors := make([]string, 0)
+
+	for count > 0 {
+		select {
+		case <-sigch:
+			signal.Stop(sigch)
+			return fmt.Errorf("unable to update: interrupted")
+		case err = <-ch:
+			if err != nil {
+				errors = append(errors, err.Error())
+			}
+			count--
+		}
+	}
+
+	context.Progress().ShutdownBar()
+	signal.Stop(sigch)
+
+	if len(errors) > 0 {
+		return fmt.Errorf("unable to update: download errors:\n  %s\n", strings.Join(errors, "\n  "))
+	}
+
+	err = context.ReOpenDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	repo.FinalizeDownload()
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
 	return err
 }
 
@@ -62,8 +173,8 @@ func makeCmdMirrorUpdate() *commander.Command {
 		Short:     "update mirror",
 		Long: `
 Updates remote mirror (downloads package files and meta information). When mirror is created,
-this command should be run for the first time to fetch mirror contents. This command could be
-run many times to get updated repository contents. If interrupted, command could be restarted safely.
+this command should be run for the first time to fetch mirror contents. This command can be
+run multiple times to get updated repository contents. If interrupted, command can be safely restarted.
 
 Example:
 
@@ -72,9 +183,11 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-update", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("force", false, "force update mirror even if it is locked by another process")
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
-	cmd.Flag.Var(&keyRings, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd
 }

cmd/package.go

@@ -0,0 +1,16 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdPackage() *commander.Command {
+	return &commander.Command{
+		UsageLine: "package",
+		Short:     "operations on packages",
+		Subcommands: []*commander.Command{
+			makeCmdPackageSearch(),
+			makeCmdPackageShow(),
+		},
+	}
+}

cmd/package_search.go

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPackageSearch(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	q, err := query.Parse(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	result := q.Query(context.CollectionFactory().PackageCollection())
+	result.ForEach(func(p *deb.Package) error {
+		context.Progress().Printf("%s\n", p)
+		return nil
+	})
+
+	return err
+}
+
+func makeCmdPackageSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageSearch,
+		UsageLine: "search <package-query>",
+		Short:     "search for packages matching query",
+		Long: `
+Command search displays list of packages in whole DB that match package query
+
+Example:
+
+    $ aptly package search '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-search", flag.ExitOnError),
+	}
+
+	return cmd
+}

cmd/package_show.go

@@ -0,0 +1,137 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"os"
+)
+
+func printReferencesTo(p *deb.Package) (err error) {
+	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  mirror %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  local repo %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+		if snapshot.RefList().Has(p) {
+			fmt.Printf("  snapshot %s\n", snapshot)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func aptlyPackageShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	q, err := query.Parse(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	withFiles := context.flags.Lookup("with-files").Value.Get().(bool)
+	withReferences := context.flags.Lookup("with-references").Value.Get().(bool)
+
+	w := bufio.NewWriter(os.Stdout)
+
+	result := q.Query(context.CollectionFactory().PackageCollection())
+
+	err = result.ForEach(func(p *deb.Package) error {
+		p.Stanza().WriteTo(w)
+		w.Flush()
+		fmt.Printf("\n")
+
+		if withFiles {
+			fmt.Printf("Files in the pool:\n")
+			for _, f := range p.Files() {
+				path, err := context.PackagePool().Path(f.Filename, f.Checksums.MD5)
+				if err != nil {
+					return err
+				}
+				fmt.Printf("  %s\n", path)
+			}
+			fmt.Printf("\n")
+		}
+
+		if withReferences {
+			fmt.Printf("References to package:\n")
+			printReferencesTo(p)
+			fmt.Printf("\n")
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdPackageShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageShow,
+		UsageLine: "show <package-query>",
+		Short:     "show details about packages matcing query",
+		Long: `
+Command shows displays detailed meta-information about packages
+matching query. Information from Debian control file is displayed.
+Optionally information about package files and
+inclusion into mirrors/snapshots/local repos is shown.
+
+Example:
+
+    $ aptly package show nginx-light_1.2.1-2.2+wheezy2_i386'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-files", false, "display information about files from package pool")
+	cmd.Flag.Bool("with-references", false, "display information about mirrors, snapshots and local repos referencing this package")
+
+	return cmd
+}

cmd/publish.go

@@ -1,19 +1,21 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"strings"
 )
 
-func getSigner(cmd *commander.Command) (utils.Signer, error) {
-	if cmd.Flag.Lookup("skip-signing").Value.Get().(bool) || utils.Config.GpgDisableSign {
+func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
+	if LookupOption(context.Config().GpgDisableSign, flags, "skip-signing") {
 		return nil, nil
 	}
 
 	signer := &utils.GpgSigner{}
-	signer.SetKey(cmd.Flag.Lookup("gpg-key").Value.String())
-	signer.SetKeyRing(cmd.Flag.Lookup("keyring").Value.String(), cmd.Flag.Lookup("secret-keyring").Value.String())
+	signer.SetKey(flags.Lookup("gpg-key").Value.String())
+	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
+	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
 
 	err := signer.Init()
 	if err != nil {
@@ -24,15 +26,31 @@ func getSigner(cmd *commander.Command) (utils.Signer, error) {
 
 }
 
+func parsePrefix(param string) (storage, prefix string) {
+	i := strings.LastIndex(param, ":")
+	if i != -1 {
+		storage = param[:i]
+		prefix = param[i+1:]
+		if prefix == "" {
+			prefix = "."
+		}
+	} else {
+		prefix = param
+	}
+	return
+}
+
 func makeCmdPublish() *commander.Command {
 	return &commander.Command{
 		UsageLine: "publish",
 		Short:     "manage published repositories",
 		Subcommands: []*commander.Command{
-			makeCmdPublishSnapshot(),
-			makeCmdPublishList(),
 			makeCmdPublishDrop(),
+			makeCmdPublishList(),
+			makeCmdPublishRepo(),
+			makeCmdPublishSnapshot(),
+			makeCmdPublishSwitch(),
+			makeCmdPublishUpdate(),
 		},
-		Flag: *flag.NewFlagSet("aptly-publish", flag.ExitOnError),
 	}
 }

cmd/publish_drop.go

@@ -2,33 +2,32 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
 )
 
 func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 1 || len(args) > 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	if len(args) == 2 {
-		prefix = args[1]
+		param = args[1]
 	}
 
-	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
+	storage, prefix := parsePrefix(param)
 
-	err = publishedCollecton.Remove(context.publishedStorage, prefix, distribution)
+	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
+		context.CollectionFactory(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	fmt.Printf("\nPublished repositroy has been removed successfully.\n")
+	context.Progress().Printf("\nPublished repository has been removed successfully.\n")
 
 	return err
 }
@@ -36,17 +35,16 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 func makeCmdPublishDrop() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishDrop,
-		UsageLine: "drop <distribution> [<prefix>]",
+		UsageLine: "drop <distribution> [[<endpoint>:]<prefix>]",
 		Short:     "remove published repository",
 		Long: `
-Command removes whatever has been published under specified <prefix> and
-<distribution> name.
+Command removes whatever has been published under specified <prefix>,
+publishing <endpoint> and <distribution> name.
 
 Example:
 
     $ aptly publish drop wheezy
 `,
-		Flag: *flag.NewFlagSet("aptly-publish-drop", flag.ExitOnError),
 	}
 
 	return cmd

cmd/publish_list.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"sort"
 )
 
@@ -12,26 +11,24 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	if publishedCollecton.Len() == 0 {
-		fmt.Printf("No snapshots have been published. Publish a snapshot by running `aptly publish snapshot ...`.\n")
-		return err
-	}
+	published := make([]string, 0, context.CollectionFactory().PublishedRepoCollection().Len())
 
-	published := make([]string, 0, publishedCollecton.Len())
-
-	err = publishedCollecton.ForEach(func(repo *debian.PublishedRepo) error {
-		err := publishedCollecton.LoadComplete(repo, snapshotCollection)
+	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
 		if err != nil {
 			return err
 		}
 
-		published = append(published, repo.String())
+		if raw {
+			published = append(published, fmt.Sprintf("%s %s", repo.StoragePrefix(), repo.Distribution))
+		} else {
+			published = append(published, repo.String())
+		}
 		return nil
 	})
 
@@ -41,10 +38,21 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 
 	sort.Strings(published)
 
-	fmt.Printf("Published repositories:\n")
+	if raw {
+		for _, info := range published {
+			fmt.Printf("%s\n", info)
+		}
+	} else {
+		if len(published) == 0 {
+			fmt.Printf("No snapshots/local repos have been published. Publish a snapshot by running `aptly publish snapshot ...`.\n")
+			return err
+		}
 
-	for _, description := range published {
-		fmt.Printf("  * %s\n", description)
+		fmt.Printf("Published repositories:\n")
+
+		for _, description := range published {
+			fmt.Printf("  * %s\n", description)
+		}
 	}
 
 	return err
@@ -62,8 +70,9 @@ Example:
 
     $ aptly publish list
 `,
-		Flag: *flag.NewFlagSet("aptly-publish-list", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+
 	return cmd
 }

cmd/publish_repo.go

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdPublishRepo() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSnapshotOrRepo,
+		UsageLine: "repo <name> [[<endpoint>:]<prefix>]",
+		Short:     "publish local repository",
+		Long: `
+Command publishes current state of local repository ready to be consumed
+by apt tools. Published repostiories appear under rootDir/public directory.
+Valid GPG key is required for publishing.
+
+Multiple component repository could be published by specifying several
+components split by commas via -component flag and multiple local
+repositories as the arguments:
+
+    aptly publish repo -component=main,contrib repo-main repo-contrib
+
+It is not recommended to publish local repositories directly unless the
+repository is for testing purposes and changes happen frequently. For
+production usage please take snapshot of repository and publish it
+using publish snapshot command.
+
+Example:
+
+    $ aptly publish repo testing
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-repo", flag.ExitOnError),
+	}
+	cmd.Flag.String("distribution", "", "distribution name to publish")
+	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+
+	return cmd
+}

cmd/publish_snapshot.go

@@ -2,124 +2,192 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"strings"
 )
 
-func aptlyPublishSnapshot(cmd *commander.Command, args []string) error {
+func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	var err error
-	if len(args) < 1 || len(args) > 2 {
+
+	components := strings.Split(context.flags.Lookup("component").Value.String(), ",")
+
+	if len(args) < len(components) || len(args) > len(components)+1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	name := args[0]
-
-	var prefix string
-	if len(args) == 2 {
-		prefix = args[1]
+	var param string
+	if len(args) == len(components)+1 {
+		param = args[len(components)]
+		args = args[0 : len(args)-1]
 	} else {
-		prefix = ""
+		param = ""
 	}
+	storage, prefix := parsePrefix(param)
 
-	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
+	var (
+		sources = []interface{}{}
+		message string
+	)
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	snapshot, err := snapshotCollection.ByName(name)
-	if err != nil {
-		return fmt.Errorf("unable to publish: %s", err)
-	}
+	if cmd.Name() == "snapshot" {
+		var (
+			snapshot     *deb.Snapshot
+			emptyWarning = false
+			parts        = []string{}
+		)
 
-	err = snapshotCollection.LoadComplete(snapshot)
-	if err != nil {
-		return fmt.Errorf("unable to publish: %s", err)
-	}
+		for _, name := range args {
+			snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
 
-	var sourceRepo *debian.RemoteRepo
+			err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
 
-	if snapshot.SourceKind == "repo" && len(snapshot.SourceIDs) == 1 {
-		repoCollection := debian.NewRemoteRepoCollection(context.database)
+			sources = append(sources, snapshot)
+			parts = append(parts, snapshot.Name)
 
-		sourceRepo, _ = repoCollection.ByUUID(snapshot.SourceIDs[0])
-	}
+			if snapshot.NumPackages() == 0 {
+				emptyWarning = true
+			}
+		}
 
-	component := cmd.Flag.Lookup("component").Value.String()
-	if component == "" {
-		if sourceRepo != nil && len(sourceRepo.Components) == 1 {
-			component = sourceRepo.Components[0]
+		if len(parts) == 1 {
+			message = fmt.Sprintf("Snapshot %s has", parts[0])
 		} else {
-			component = "main"
+			message = fmt.Sprintf("Snapshots %s have", strings.Join(parts, ", "))
+
 		}
+
+		if emptyWarning {
+			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
+		}
+	} else if cmd.Name() == "repo" {
+		var (
+			localRepo    *deb.LocalRepo
+			emptyWarning = false
+			parts        = []string{}
+		)
+
+		for _, name := range args {
+			localRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			err = context.CollectionFactory().LocalRepoCollection().LoadComplete(localRepo)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			sources = append(sources, localRepo)
+			parts = append(parts, localRepo.Name)
+
+			if localRepo.NumPackages() == 0 {
+				emptyWarning = true
+			}
+		}
+
+		if len(parts) == 1 {
+			message = fmt.Sprintf("Local repo %s has", parts[0])
+		} else {
+			message = fmt.Sprintf("Local repos %s have", strings.Join(parts, ", "))
+
+		}
+
+		if emptyWarning {
+			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
+		}
+	} else {
+		panic("unknown command")
 	}
 
-	distribution := cmd.Flag.Lookup("distribution").Value.String()
-	if distribution == "" {
-		if sourceRepo != nil {
-			distribution = sourceRepo.Distribution
-		}
+	distribution := context.flags.Lookup("distribution").Value.String()
 
-		if distribution == "" {
-			return fmt.Errorf("unable to guess distribution name, please specify explicitly")
-		}
-	}
-
-	published, err := debian.NewPublishedRepo(prefix, distribution, component, context.architecturesList, snapshot)
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
+	published.Origin = cmd.Flag.Lookup("origin").Value.String()
+	published.Label = cmd.Flag.Lookup("label").Value.String()
 
-	duplicate := publishedCollecton.CheckDuplicate(published)
+	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
-		publishedCollecton.LoadComplete(duplicate, snapshotCollection)
+		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
 		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 	}
 
-	signer, err := getSigner(cmd)
+	signer, err := getSigner(context.flags)
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	packageCollection := debian.NewPackageCollection(context.database)
-	err = published.Publish(context.packagePool, context.publishedStorage, packageCollection, signer, context.progress)
+	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = publishedCollecton.Add(published)
+	err = context.CollectionFactory().PublishedRepoCollection().Add(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	if prefix != "" && !strings.HasSuffix(prefix, "/") {
+	var repoComponents string
+	prefix, repoComponents, distribution = published.Prefix, strings.Join(published.Components(), " "), published.Distribution
+	if prefix == "." {
+		prefix = ""
+	} else if !strings.HasSuffix(prefix, "/") {
 		prefix += "/"
 	}
 
-	context.progress.Printf("\nSnapshot %s has been successfully published.\nPlease setup your webserver to serve directory '%s' with autoindexing.\n",
-		snapshot.Name, context.publishedStorage.PublicPath())
-	context.progress.Printf("Now you can add following line to apt sources:\n")
-	context.progress.Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, component)
-	if utils.StrSliceHasItem(published.Architectures, "source") {
-		context.progress.Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, component)
+	context.Progress().Printf("\n%s been successfully published.\n", message)
+
+	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.LocalPublishedStorage); ok {
+		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
+			localStorage.PublicPath())
 	}
-	context.progress.Printf("Don't forget to add your GPG key to apt with apt-key.\n")
-	context.progress.Printf("\nYou can also use `aptly serve` to publish your repositories over HTTP quickly.\n")
+
+	context.Progress().Printf("Now you can add following line to apt sources:\n")
+	context.Progress().Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
+	if utils.StrSliceHasItem(published.Architectures, "source") {
+		context.Progress().Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
+	}
+	context.Progress().Printf("Don't forget to add your GPG key to apt with apt-key.\n")
+	context.Progress().Printf("\nYou can also use `aptly serve` to publish your repositories over HTTP quickly.\n")
 
 	return err
 }
 
 func makeCmdPublishSnapshot() *commander.Command {
 	cmd := &commander.Command{
-		Run:       aptlyPublishSnapshot,
-		UsageLine: "snapshot <name> [<prefix>]",
+		Run:       aptlyPublishSnapshotOrRepo,
+		UsageLine: "snapshot <name> [[<endpoint>:]<prefix>]",
 		Short:     "publish snapshot",
 		Long: `
-Command publish publishes snapshot as Debian repository ready to be consumed
+Command publishes snapshot as Debian repository ready to be consumed
 by apt tools. Published repostiories appear under rootDir/public directory.
 Valid GPG key is required for publishing.
 
+Multiple component repository could be published by specifying several
+components split by commas via -component flag and multiple snapshots
+as the arguments:
+
+    aptly publish snapshot -component=main,contrib snap-main snap-contrib
+
 Example:
 
     $ aptly publish snapshot wheezy-main
@@ -127,11 +195,16 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-publish-snapshot", flag.ExitOnError),
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
-	cmd.Flag.String("component", "", "component name to publish")
+	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
-	cmd.Flag.String("keyring", "", "GPG keyring to use (instead of default)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd
 }

cmd/publish_switch.go

@@ -0,0 +1,141 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"strings"
+)
+
+func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
+	var err error
+
+	components := strings.Split(context.flags.Lookup("component").Value.String(), ",")
+
+	if len(args) < len(components)+1 || len(args) > len(components)+2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	param := "."
+
+	var (
+		names    []string
+		snapshot *deb.Snapshot
+	)
+
+	if len(args) == len(components)+2 {
+		param = args[1]
+		names = args[2:]
+	} else {
+		names = args[1:]
+	}
+
+	storage, prefix := parsePrefix(param)
+
+	var published *deb.PublishedRepo
+
+	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	if published.SourceKind != "snapshot" {
+		return fmt.Errorf("unable to update: not a snapshot publish")
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	publishedComponents := published.Components()
+	if len(components) == 1 && len(publishedComponents) == 1 && components[0] == "" {
+		components = publishedComponents
+	}
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names))
+	}
+
+	for i, component := range components {
+		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(names[i])
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
+
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
+
+		published.UpdateSnapshot(component, snapshot)
+	}
+
+	signer, err := getSigner(context.flags)
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG signer: %s", err)
+	}
+
+	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
+
+	return err
+}
+
+func makeCmdPublishSwitch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSwitch,
+		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
+		Short:     "update published repository by switching to new snapshot",
+		Long: `
+Command switches in-place published repository with new snapshot contents. All
+publishing parameters are preserved (architecture list, distribution,
+component).
+
+For multiple component repositories, flag -component should be given with
+list of components to update. Corresponding snapshots should be given in the
+same order, e.g.:
+
+	aptly publish update -component=main,contrib wheezy wh-main wh-contrib
+
+Example:
+
+    $ aptly publish update wheezy ppa wheezy-7.5
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
+	}
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+
+	return cmd
+}

cmd/publish_update.go

@@ -0,0 +1,107 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+	storage, prefix := parsePrefix(param)
+
+	var published *deb.PublishedRepo
+
+	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	if published.SourceKind != "local" {
+		return fmt.Errorf("unable to update: not a local repository publish")
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	components := published.Components()
+	for _, component := range components {
+		published.UpdateLocalRepo(component)
+	}
+
+	signer, err := getSigner(context.flags)
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG signer: %s", err)
+	}
+
+	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
+
+	return err
+}
+
+func makeCmdPublishUpdate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishUpdate,
+		UsageLine: "update <distribution> [[<endpoint>:]<prefix>]",
+		Short:     "update published local repository",
+		Long: `
+Command re-publishes (updates) published local repository. <distribution>
+and <prefix> should be occupied with local repository published
+using command aptly publish repo. Update happens in-place with
+minimum possible downtime for published repository.
+
+For multiple component published repositories, all local repositories
+are updated.
+
+Example:
+
+    $ aptly publish update wheezy ppa
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError),
+	}
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+
+	return cmd
+}

cmd/repo.go

@@ -1,8 +1,7 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
 )
 
 func makeCmdRepo() *commander.Command {
@@ -14,12 +13,14 @@ func makeCmdRepo() *commander.Command {
 			makeCmdRepoCopy(),
 			makeCmdRepoCreate(),
 			makeCmdRepoDrop(),
+			makeCmdRepoEdit(),
 			makeCmdRepoImport(),
 			makeCmdRepoList(),
 			makeCmdRepoMove(),
 			makeCmdRepoRemove(),
 			makeCmdRepoShow(),
+			makeCmdRepoRename(),
+			makeCmdRepoSearch(),
 		},
-		Flag: *flag.NewFlagSet("aptly-repo", flag.ExitOnError),
 	}
 }

cmd/repo_add.go

@@ -2,10 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"os"
 	"path/filepath"
 	"sort"
@@ -16,60 +16,65 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
 	verifier := &utils.GpgVerifier{}
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-	repo, err := localRepoCollection.ByName(name)
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
-	err = localRepoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
-	context.progress.Printf("Loading packages...\n")
+	context.Progress().Printf("Loading packages...\n")
 
-	packageCollection := debian.NewPackageCollection(context.database)
-	list, err := debian.NewPackageListFromRefList(repo.RefList(), packageCollection, context.progress)
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
+	forceReplace := context.flags.Lookup("force-replace").Value.Get().(bool)
+
 	packageFiles := []string{}
+	failedFiles := []string{}
 
 	for _, location := range args[1:] {
-		info, err := os.Stat(location)
-		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to process %s: %s@|", location, err)
+		info, err2 := os.Stat(location)
+		if err2 != nil {
+			context.Progress().ColoredPrintf("@y[!]@| @!Unable to process %s: %s@|", location, err2)
+			failedFiles = append(failedFiles, location)
 			continue
 		}
 		if info.IsDir() {
-			err = filepath.Walk(location, func(path string, info os.FileInfo, err error) error {
-				if err != nil {
-					return err
+			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
+				if err3 != nil {
+					return err3
 				}
 				if info.IsDir() {
 					return nil
 				}
 
-				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
+				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+					strings.HasSuffix(info.Name(), ".dsc") {
 					packageFiles = append(packageFiles, path)
 				}
 
 				return nil
 			})
 		} else {
-			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
+			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+				strings.HasSuffix(info.Name(), ".dsc") {
 				packageFiles = append(packageFiles, location)
 			} else {
-				context.progress.ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
+				context.Progress().ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
+				failedFiles = append(failedFiles, location)
 				continue
 			}
 		}
@@ -78,48 +83,59 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 	processedFiles := []string{}
 	sort.Strings(packageFiles)
 
+	if forceReplace {
+		list.PrepareIndex()
+	}
+
 	for _, file := range packageFiles {
 		var (
-			stanza debian.Stanza
-			err    error
-			p      *debian.Package
+			stanza deb.Stanza
+			p      *deb.Package
 		)
 
 		candidateProcessedFiles := []string{}
 		isSourcePackage := strings.HasSuffix(file, ".dsc")
+		isUdebPackage := strings.HasSuffix(file, ".udeb")
 
 		if isSourcePackage {
-			stanza, err = debian.GetControlFileFromDsc(file, verifier)
+			stanza, err = deb.GetControlFileFromDsc(file, verifier)
 
 			if err == nil {
 				stanza["Package"] = stanza["Source"]
 				delete(stanza, "Source")
 
-				p, err = debian.NewSourcePackageFromControlFile(stanza)
+				p, err = deb.NewSourcePackageFromControlFile(stanza)
 			}
 		} else {
-			stanza, err = debian.GetControlFileFromDeb(file)
-			p = debian.NewPackageFromControlFile(stanza)
+			stanza, err = deb.GetControlFileFromDeb(file)
+			if isUdebPackage {
+				p = deb.NewUdebPackageFromControlFile(stanza)
+			} else {
+				p = deb.NewPackageFromControlFile(stanza)
+			}
 		}
 		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
+			context.Progress().ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
+			failedFiles = append(failedFiles, file)
 			continue
 		}
 
-		checksums, err := utils.ChecksumsForFile(file)
+		var checksums utils.ChecksumInfo
+		checksums, err = utils.ChecksumsForFile(file)
 		if err != nil {
 			return err
 		}
 
 		if isSourcePackage {
-			p.UpdateFiles(append(p.Files(), debian.PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
+			p.UpdateFiles(append(p.Files(), deb.PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
 		} else {
-			p.UpdateFiles([]debian.PackageFile{debian.PackageFile{Filename: filepath.Base(file), Checksums: checksums}})
+			p.UpdateFiles([]deb.PackageFile{deb.PackageFile{Filename: filepath.Base(file), Checksums: checksums}})
 		}
 
-		err = context.packagePool.Import(file, checksums.MD5)
+		err = context.PackagePool().Import(file, checksums.MD5)
 		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", file, err)
+			context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", file, err)
+			failedFiles = append(failedFiles, file)
 			continue
 		}
 
@@ -131,9 +147,10 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 				continue
 			}
 			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
-			err = context.packagePool.Import(sourceFile, f.Checksums.MD5)
+			err = context.PackagePool().Import(sourceFile, f.Checksums.MD5)
 			if err != nil {
-				context.progress.ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", sourceFile, err)
+				context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", sourceFile, err)
+				failedFiles = append(failedFiles, file)
 				break
 			}
 
@@ -144,30 +161,40 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 			continue
 		}
 
-		err = packageCollection.Update(p)
+		err = context.CollectionFactory().PackageCollection().Update(p)
 		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to save package %s: %s@|", p, err)
+			context.Progress().ColoredPrintf("@y[!]@| @!Unable to save package %s: %s@|", p, err)
+			failedFiles = append(failedFiles, file)
 			continue
 		}
 
+		if forceReplace {
+			conflictingPackages := list.Search(deb.Dependency{Pkg: p.Name, Version: p.Version, Architecture: p.Architecture}, true)
+			for _, cp := range conflictingPackages {
+				context.Progress().ColoredPrintf("@r[-]@| %s removed due to conflict with package being added", cp)
+				list.Remove(cp)
+			}
+		}
+
 		err = list.Add(p)
 		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
+			context.Progress().ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
+			failedFiles = append(failedFiles, file)
 			continue
 		}
 
-		context.progress.ColoredPrintf("@g[+]@| %s added@|", p)
+		context.Progress().ColoredPrintf("@g[+]@| %s added@|", p)
 		processedFiles = append(processedFiles, candidateProcessedFiles...)
 	}
 
-	repo.UpdateRefList(debian.NewPackageRefListFromPackageList(list))
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-	err = localRepoCollection.Update(repo)
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to save: %s", err)
 	}
 
-	if cmd.Flag.Lookup("remove-files").Value.Get().(bool) {
+	if context.flags.Lookup("remove-files").Value.Get().(bool) {
 		processedFiles = utils.StrSliceDeduplicate(processedFiles)
 
 		for _, file := range processedFiles {
@@ -178,6 +205,15 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		}
 	}
 
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
 	return err
 }
 
@@ -187,11 +223,11 @@ func makeCmdRepoAdd() *commander.Command {
 		UsageLine: "add <name> <package file.deb>|<directory> ...",
 		Short:     "add packages to local repository",
 		Long: `
-Command adds packages to local repository from .deb (binary packages) and .dsc (source packages) files.
-When importing from directory aptly would do recursive scan looking for all files matching *.deb or *.dsc
-patterns. Every file discovered would be analyzed to extract metadata, package would be created and added
-to database. Files would be imported to internal package pool. For source packages, all required files are
-added as well automatically. Extra files for source package should be in the same directory as *.dsc file.
+Command adds packages to local repository from .deb, .udeb (binary packages) and .dsc (source packages) files.
+When importing from directory aptly would do recursive scan looking for all files matching *.[u]deb or *.dsc
+patterns. Every file discovered would be analyzed to extract metadata, package would then be created and added
+to the database. Files would be imported to internal package pool. For source packages, all required files are
+added automatically as well. Extra files for source package should be in the same directory as *.dsc file.
 
 Example:
 
@@ -201,6 +237,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("remove-files", false, "remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
 
 	return cmd
 }

cmd/repo_copy.go

@@ -1,17 +1,17 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func makeCmdRepoCopy() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "copy <src-name> <dst-name> <package-spec> ...",
+		UsageLine: "copy <src-name> <dst-name> <package-query> ...",
 		Short:     "copy packages between local repositories",
 		Long: `
-Command copy copies packages matching <package-spec> from local repo
+Command copy copies packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
 Example:

cmd/repo_create.go

@@ -2,23 +2,23 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	repo := debian.NewLocalRepo(args[0], cmd.Flag.Lookup("comment").Value.String())
+	repo := deb.NewLocalRepo(args[0], context.flags.Lookup("comment").Value.String())
+	repo.DefaultDistribution = context.flags.Lookup("distribution").Value.String()
+	repo.DefaultComponent = context.flags.Lookup("component").Value.String()
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-
-	err = localRepoCollection.Add(repo)
+	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add local repo: %s", err)
 	}
@@ -45,6 +45,8 @@ Example:
 	}
 
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
+	cmd.Flag.String("distribution", "", "default distribution when publishing")
+	cmd.Flag.String("component", "main", "default component when publishing")
 
 	return cmd
 }

cmd/repo_drop.go

@@ -2,30 +2,41 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-	repo, err := localRepoCollection.ByName(name)
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	force := cmd.Flag.Lookup("force").Value.Get().(bool)
+	published := context.CollectionFactory().PublishedRepoCollection().ByLocalRepo(repo)
+	if len(published) > 0 {
+		fmt.Printf("Local repo `%s` is published currently:\n", repo.Name)
+		for _, repo := range published {
+			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+			if err != nil {
+				return fmt.Errorf("unable to load published: %s", err)
+			}
+			fmt.Printf(" * %s\n", repo)
+		}
+
+		return fmt.Errorf("unable to drop: local repo is published")
+	}
+
+	force := context.flags.Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshotCollection := debian.NewSnapshotCollection(context.database)
-		snapshots := snapshotCollection.ByLocalRepoSource(repo)
+		snapshots := context.CollectionFactory().SnapshotCollection().ByLocalRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Local repo `%s` was used to create following snapshots:\n", repo.Name)
@@ -37,7 +48,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = localRepoCollection.Drop(repo)
+	err = context.CollectionFactory().LocalRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
@@ -53,8 +64,8 @@ func makeCmdRepoDrop() *commander.Command {
 		UsageLine: "drop <name>",
 		Short:     "delete local repository",
 		Long: `
-Drop deletes information about local repo. Package data is not deleted
-(it could be still used by other mirrors or snapshots).
+Drop information about deletions from local repo. Package data is not deleted
+(since it could be still used by other mirrors or snapshots).
 
 Example:
 

cmd/repo_edit.go

@@ -0,0 +1,68 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoEdit(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	if context.flags.Lookup("comment").Value.String() != "" {
+		repo.Comment = context.flags.Lookup("comment").Value.String()
+	}
+
+	if context.flags.Lookup("distribution").Value.String() != "" {
+		repo.DefaultDistribution = context.flags.Lookup("distribution").Value.String()
+	}
+
+	if context.flags.Lookup("component").Value.String() != "" {
+		repo.DefaultComponent = context.flags.Lookup("component").Value.String()
+	}
+
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fmt.Printf("Local repo %s successfully updated.\n", repo)
+	return err
+}
+
+func makeCmdRepoEdit() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoEdit,
+		UsageLine: "edit <name>",
+		Short:     "edit properties of local repository",
+		Long: `
+Command edit allows one to change metadata of local repository:
+comment, default distribution and component.
+
+Example:
+
+  $ aptly repo edit -distribution=wheezy testing
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-edit", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
+	cmd.Flag.String("distribution", "", "default distribution when publishing")
+	cmd.Flag.String("component", "", "default component when publishing")
+
+	return cmd
+}

cmd/repo_import.go

@@ -1,17 +1,17 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func makeCmdRepoImport() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "import <src-mirror> <dst-repo> <package-spec> ...",
+		UsageLine: "import <src-mirror> <dst-repo> <package-query> ...",
 		Short:     "import packages from mirror to local repository",
 		Long: `
-Command import looks up packages matching <package-spec> in mirror <src-mirror>
+Command import looks up packages matching <package-query> in mirror <src-mirror>
 and copies them to local repo <dst-repo>.
 
 Example:

cmd/repo_list.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"sort"
 )
 
@@ -12,35 +11,47 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	if localRepoCollection.Len() > 0 {
-		fmt.Printf("List of mirrors:\n")
-		repos := make([]string, localRepoCollection.Len())
-		i := 0
-		localRepoCollection.ForEach(func(repo *debian.LocalRepo) error {
-			err := localRepoCollection.LoadComplete(repo)
+	repos := make([]string, context.CollectionFactory().LocalRepoCollection().Len())
+	i := 0
+	context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		if raw {
+			repos[i] = repo.Name
+		} else {
+			err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 			if err != nil {
 				return err
 			}
 
 			repos[i] = fmt.Sprintf(" * %s (packages: %d)", repo.String(), repo.NumPackages())
-			i++
-			return nil
-		})
-
-		sort.Strings(repos)
-		for _, repo := range repos {
-			fmt.Println(repo)
 		}
+		i++
+		return nil
+	})
 
-		fmt.Printf("\nTo get more information about local repository, run `aptly repo show <name>`.\n")
+	sort.Strings(repos)
+
+	if raw {
+		for _, repo := range repos {
+			fmt.Printf("%s\n", repo)
+		}
 	} else {
-		fmt.Printf("No local repositories found, create one with `aptly repo create ...`.\n")
+		if len(repos) > 0 {
+			fmt.Printf("List of local repos:\n")
+			for _, repo := range repos {
+				fmt.Println(repo)
+			}
+
+			fmt.Printf("\nTo get more information about local repository, run `aptly repo show <name>`.\n")
+		} else {
+			fmt.Printf("No local repositories found, create one with `aptly repo create ...`.\n")
+		}
 	}
+
 	return err
 }
 
@@ -50,14 +61,15 @@ func makeCmdRepoList() *commander.Command {
 		UsageLine: "list",
 		Short:     "list local repositories",
 		Long: `
-List shows full list of local package repositories.
+List command shows full list of local package repositories.
 
 Example:
 
   $ aptly repo list
 `,
-		Flag: *flag.NewFlagSet("aptly-repo-list", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+
 	return cmd
 }

cmd/repo_move.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"sort"
 )
 
@@ -12,30 +13,28 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 3 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	command := cmd.Name()
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-
-	dstRepo, err := localRepoCollection.ByName(args[1])
+	dstRepo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	err = localRepoCollection.LoadComplete(dstRepo)
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(dstRepo)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
 	var (
-		srcRefList *debian.PackageRefList
-		srcRepo    *debian.LocalRepo
+		srcRefList *deb.PackageRefList
+		srcRepo    *deb.LocalRepo
 	)
 
 	if command == "copy" || command == "move" {
-		srcRepo, err = localRepoCollection.ByName(args[0])
+		srcRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -44,43 +43,42 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to %s: source and destination are the same", command)
 		}
 
-		err = localRepoCollection.LoadComplete(srcRepo)
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(srcRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
 		srcRefList = srcRepo.RefList()
 	} else if command == "import" {
-		repoCollection := debian.NewRemoteRepoCollection(context.database)
+		var srcRemoteRepo *deb.RemoteRepo
 
-		srcRepo, err := repoCollection.ByName(args[0])
+		srcRemoteRepo, err = context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		err = repoCollection.LoadComplete(srcRepo)
+		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(srcRemoteRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		if srcRepo.RefList() == nil {
+		if srcRemoteRepo.RefList() == nil {
 			return fmt.Errorf("unable to %s: mirror not updated", command)
 		}
 
-		srcRefList = srcRepo.RefList()
+		srcRefList = srcRemoteRepo.RefList()
 	} else {
 		panic("unexpected command")
 	}
 
-	context.progress.Printf("Loading packages...\n")
+	context.Progress().Printf("Loading packages...\n")
 
-	packageCollection := debian.NewPackageCollection(context.database)
-	dstList, err := debian.NewPackageListFromRefList(dstRepo.RefList(), packageCollection, context.progress)
+	dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	srcList, err := debian.NewPackageListFromRefList(srcRefList, packageCollection, context.progress)
+	srcList, err := deb.NewPackageListFromRefList(srcRefList, context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -89,14 +87,14 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	var architecturesList []string
 
-	withDeps := cmd.Flag.Lookup("with-deps").Value.Get().(bool)
+	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
 
 	if withDeps {
 		dstList.PrepareIndex()
 
 		// Calculate architectures
-		if len(context.architecturesList) > 0 {
-			architecturesList = context.architecturesList
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
 		} else {
 			architecturesList = dstList.Architectures(false)
 		}
@@ -108,7 +106,15 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	toProcess, err := srcList.Filter(args[2:], withDeps, dstList, context.dependencyOptions, architecturesList)
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i := 0; i < len(args)-2; i++ {
+		queries[i], err = query.Parse(args[i+2])
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+	}
+
+	toProcess, err := srcList.Filter(queries, withDeps, dstList, context.DependencyOptions(), architecturesList)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
@@ -123,7 +129,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		verb = "imported"
 	}
 
-	err = toProcess.ForEach(func(p *debian.Package) error {
+	err = toProcess.ForEach(func(p *deb.Package) error {
 		err = dstList.Add(p)
 		if err != nil {
 			return err
@@ -132,27 +138,27 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		if command == "move" {
 			srcList.Remove(p)
 		}
-		context.progress.ColoredPrintf("@g[o]@| %s %s", p, verb)
+		context.Progress().ColoredPrintf("@g[o]@| %s %s", p, verb)
 		return nil
 	})
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	if cmd.Flag.Lookup("dry-run").Value.Get().(bool) {
-		context.progress.Printf("\nChanges not saved, as dry run has been requested.\n")
+	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
-		dstRepo.UpdateRefList(debian.NewPackageRefListFromPackageList(dstList))
+		dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
 
-		err = localRepoCollection.Update(dstRepo)
+		err = context.CollectionFactory().LocalRepoCollection().Update(dstRepo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
 
 		if command == "move" {
-			srcRepo.UpdateRefList(debian.NewPackageRefListFromPackageList(srcList))
+			srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
 
-			err = localRepoCollection.Update(srcRepo)
+			err = context.CollectionFactory().LocalRepoCollection().Update(srcRepo)
 			if err != nil {
 				return fmt.Errorf("unable to save: %s", err)
 			}
@@ -165,10 +171,10 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 func makeCmdRepoMove() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "move <src-name> <dst-name> <package-spec> ...",
+		UsageLine: "move <src-name> <dst-name> <package-query> ...",
 		Short:     "move packages between local repositories",
 		Long: `
-Command move moves packages matching <package-spec> from local repo
+Command move moves packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
 Example:

cmd/repo_remove.go

@@ -2,57 +2,64 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-	repo, err := localRepoCollection.ByName(name)
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	err = localRepoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	context.progress.Printf("Loading packages...\n")
+	context.Progress().Printf("Loading packages...\n")
 
-	packageCollection := debian.NewPackageCollection(context.database)
-	list, err := debian.NewPackageListFromRefList(repo.RefList(), packageCollection, context.progress)
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
+	queries := make([]deb.PackageQuery, len(args)-1)
+	for i := 0; i < len(args)-1; i++ {
+		queries[i], err = query.Parse(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to remove: %s", err)
+		}
+	}
+
 	list.PrepareIndex()
-	toRemove, err := list.Filter(args[1:], false, nil, 0, nil)
+	toRemove, err := list.Filter(queries, false, nil, 0, nil)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	toRemove.ForEach(func(p *debian.Package) error {
+	toRemove.ForEach(func(p *deb.Package) error {
 		list.Remove(p)
-		context.progress.ColoredPrintf("@r[-]@| %s removed", p)
+		context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
 		return nil
 	})
 
-	if cmd.Flag.Lookup("dry-run").Value.Get().(bool) {
-		context.progress.Printf("\nChanges not saved, as dry run has been requested.\n")
+	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
-		repo.UpdateRefList(debian.NewPackageRefListFromPackageList(list))
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = localRepoCollection.Update(repo)
+		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
@@ -64,10 +71,10 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 func makeCmdRepoRemove() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoRemove,
-		UsageLine: "remove <name> <package-spec> ...",
+		UsageLine: "remove <name> <package-query> ...",
 		Short:     "remove packages from local repository",
 		Long: `
-Commands removes packages matching <package-spec> from local repository
+Commands removes packages matching <package-query> from local repository
 <name>. If removed packages are not referenced by other repos or
 snapshots, they can be removed completely (including files) by running
 'aptly db cleanup'.

cmd/repo_rename.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyRepoRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.LocalRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	repo, err = context.CollectionFactory().LocalRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().LocalRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: local repo %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nLocal repo %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdRepoRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames local repository",
+		Long: `
+Command changes name of the local repo. Local repo name should be unique.
+
+Example:
+
+  $ aptly repo rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/repo_search.go

@@ -0,0 +1,26 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdRepoSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> <package-query>",
+		Short:     "search repo for packages matching query",
+		Long: `
+Command search displays list of packages in local repository that match package query
+
+Example:
+
+    $ aptly repo search my-software '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+
+	return cmd
+}

cmd/repo_show.go

@@ -2,36 +2,36 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-	repo, err := localRepoCollection.ByName(name)
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = localRepoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
 	fmt.Printf("Name: %s\n", repo.Name)
 	fmt.Printf("Comment: %s\n", repo.Comment)
+	fmt.Printf("Default Distribution: %s\n", repo.DefaultDistribution)
+	fmt.Printf("Default Component: %s\n", repo.DefaultComponent)
 	fmt.Printf("Number of packages: %d\n", repo.NumPackages())
 
-	withPackages := cmd.Flag.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		ListPackagesRefList(repo.RefList())
 	}
@@ -45,7 +45,7 @@ func makeCmdRepoShow() *commander.Command {
 		UsageLine: "show <name>",
 		Short:     "show details about local repository",
 		Long: `
-Show shows full information about local package repository.
+Show command shows full information about local package repository.
 
 ex:
   $ aptly repo show testing

cmd/run.go

@@ -0,0 +1,44 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/commander"
+)
+
+// Run runs single command starting from root cmd with args, optionally initializing context
+func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode int) {
+	defer func() {
+		if r := recover(); r != nil {
+			fatal, ok := r.(*FatalError)
+			if !ok {
+				panic(r)
+			}
+			fmt.Println("ERROR:", fatal.Message)
+			returnCode = fatal.ReturnCode
+		}
+	}()
+
+	returnCode = 0
+
+	flags, args, err := cmd.ParseFlags(cmdArgs)
+	if err != nil {
+		Fatal(err)
+	}
+
+	if initContext {
+		err = InitContext(flags)
+		if err != nil {
+			Fatal(err)
+		}
+		defer ShutdownContext()
+	}
+
+	context.UpdateFlags(flags)
+
+	err = cmd.Dispatch(args)
+	if err != nil {
+		Fatal(err)
+	}
+
+	return
+}

cmd/serve.go

@@ -2,28 +2,32 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"net"
 	"net/http"
 	"os"
 	"sort"
+	"strings"
 )
 
 func aptlyServe(cmd *commander.Command, args []string) error {
 	var err error
 
-	publishedCollection := debian.NewPublishedRepoCollection(context.database)
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
 
-	if publishedCollection.Len() == 0 {
+	if context.CollectionFactory().PublishedRepoCollection().Len() == 0 {
 		fmt.Printf("No published repositories, unable to serve.\n")
 		return nil
 	}
 
-	listen := cmd.Flag.Lookup("listen").Value.String()
+	listen := context.flags.Lookup("listen").Value.String()
 
 	listenHost, listenPort, err := net.SplitHostPort(listen)
 
@@ -40,11 +44,11 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Serving published repositories, recommended apt sources list:\n\n")
 
-	sources := make(sort.StringSlice, 0, publishedCollection.Len())
-	published := make(map[string]*debian.PublishedRepo, publishedCollection.Len())
+	sources := make(sort.StringSlice, 0, context.CollectionFactory().PublishedRepoCollection().Len())
+	published := make(map[string]*deb.PublishedRepo, context.CollectionFactory().PublishedRepoCollection().Len())
 
-	err = publishedCollection.ForEach(func(repo *debian.PublishedRepo) error {
-		err := publishedCollection.LoadComplete(repo, snapshotCollection)
+	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
 		if err != nil {
 			return err
 		}
@@ -72,19 +76,20 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		}
 
 		fmt.Printf("# %s\ndeb http://%s:%s/%s %s %s\n",
-			repo, listenHost, listenPort, prefix, repo.Distribution, repo.Component)
+			repo, listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 
 		if utils.StrSliceHasItem(repo.Architectures, "source") {
 			fmt.Printf("deb-src http://%s:%s/%s %s %s\n",
-				listenHost, listenPort, prefix, repo.Distribution, repo.Component)
+				listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 		}
 	}
 
-	context.database.Close()
+	publicPath := context.GetPublishedStorage("").(aptly.LocalPublishedStorage).PublicPath()
+	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
 
-	err = http.ListenAndServe(listen, http.FileServer(http.Dir(context.publishedStorage.PublicPath())))
+	err = http.ListenAndServe(listen, http.FileServer(http.Dir(publicPath)))
 	if err != nil {
 		return fmt.Errorf("unable to serve: %s", err)
 	}

cmd/snapshot.go

@@ -1,8 +1,7 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
 )
 
 func makeCmdSnapshot() *commander.Command {
@@ -18,7 +17,9 @@ func makeCmdSnapshot() *commander.Command {
 			makeCmdSnapshotDiff(),
 			makeCmdSnapshotMerge(),
 			makeCmdSnapshotDrop(),
+			makeCmdSnapshotRename(),
+			makeCmdSnapshotSearch(),
+			makeCmdSnapshotFilter(),
 		},
-		Flag: *flag.NewFlagSet("aptly-snapshot", flag.ExitOnError),
 	}
 }

cmd/snapshot_create.go

@@ -2,52 +2,58 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 	var (
 		err      error
-		snapshot *debian.Snapshot
+		snapshot *deb.Snapshot
 	)
 
 	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" {
 		// aptly snapshot create snap from mirror mirror
+		var repo *deb.RemoteRepo
+
 		repoName, snapshotName := args[3], args[0]
 
-		repoCollection := debian.NewRemoteRepoCollection(context.database)
-		repo, err := repoCollection.ByName(repoName)
+		repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(repoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = repoCollection.LoadComplete(repo)
+		err = repo.CheckLock()
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		snapshot, err = debian.NewSnapshotFromRepository(snapshotName, repo)
+		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		snapshot, err = deb.NewSnapshotFromRepository(snapshotName, repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" {
 		// aptly snapshot create snap from repo repo
+		var repo *deb.LocalRepo
+
 		localRepoName, snapshotName := args[3], args[0]
 
-		localRepoCollection := debian.NewLocalRepoCollection(context.database)
-		repo, err := localRepoCollection.ByName(localRepoName)
+		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(localRepoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = localRepoCollection.LoadComplete(repo)
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		snapshot, err = debian.NewSnapshotFromLocalRepo(snapshotName, repo)
+		snapshot, err = deb.NewSnapshotFromLocalRepo(snapshotName, repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -55,17 +61,15 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		// aptly snapshot create snap empty
 		snapshotName := args[0]
 
-		packageList := debian.NewPackageList()
+		packageList := deb.NewPackageList()
 
-		snapshot = debian.NewSnapshotFromPackageList(snapshotName, nil, packageList, "Created as empty")
+		snapshot = deb.NewSnapshotFromPackageList(snapshotName, nil, packageList, "Created as empty")
 	} else {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-
-	err = snapshotCollection.Add(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().Add(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to add snapshot: %s", err)
 	}
@@ -97,7 +101,6 @@ Example:
 
   $ aptly snapshot create wheezy-main-today from mirror wheezy-main
 `,
-		Flag: *flag.NewFlagSet("aptly-snapshot-create", flag.ExitOnError),
 	}
 
 	return cmd

cmd/snapshot_diff.go

@@ -2,55 +2,51 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	onlyMatching := cmd.Flag.Lookup("only-matching").Value.Get().(bool)
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	packageCollection := debian.NewPackageCollection(context.database)
+	onlyMatching := context.flags.Lookup("only-matching").Value.Get().(bool)
 
 	// Load <name-a> snapshot
-	snapshotA, err := snapshotCollection.ByName(args[0])
+	snapshotA, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(snapshotA)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotA)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
 	// Load <name-b> snapshot
-	snapshotB, err := snapshotCollection.ByName(args[1])
+	snapshotB, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(snapshotB)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotB)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
 	// Calculate diff
-	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), packageCollection)
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
 	if err != nil {
 		return fmt.Errorf("unable to calculate diff: %s", err)
 	}
 
 	if len(diff) == 0 {
-		context.progress.Printf("Snapshots are identical.\n")
+		context.Progress().Printf("Snapshots are identical.\n")
 	} else {
-		context.progress.Printf("  Arch   | Package                                  | Version in A                             | Version in B\n")
+		context.Progress().Printf("  Arch   | Package                                  | Version in A                             | Version in B\n")
 		for _, pdiff := range diff {
 			if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) {
 				continue
@@ -84,7 +80,7 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 				}
 			}
 
-			context.progress.ColoredPrintf(code+" %-6s | %-40s | %-40s | %-40s", arch, pkg, verA, verB)
+			context.Progress().ColoredPrintf(code+" %-6s | %-40s | %-40s | %-40s", arch, pkg, verA, verB)
 		}
 	}
 

cmd/snapshot_drop.go

@@ -2,33 +2,30 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	snapshot, err := snapshotCollection.ByName(name)
+	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	publishedRepoCollection := debian.NewPublishedRepoCollection(context.database)
-	published := publishedRepoCollection.BySnapshot(snapshot)
+	published := context.CollectionFactory().PublishedRepoCollection().BySnapshot(snapshot)
 
 	if len(published) > 0 {
 		fmt.Printf("Snapshot `%s` is published currently:\n", snapshot.Name)
 		for _, repo := range published {
-			err = publishedRepoCollection.LoadComplete(repo, snapshotCollection)
+			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
 			if err != nil {
 				return fmt.Errorf("unable to load published: %s", err)
 			}
@@ -38,9 +35,9 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: snapshot is published")
 	}
 
-	force := cmd.Flag.Lookup("force").Value.Get().(bool)
+	force := context.flags.Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := snapshotCollection.BySnapshotSource(snapshot)
+		snapshots := context.CollectionFactory().SnapshotCollection().BySnapshotSource(snapshot)
 		if len(snapshots) > 0 {
 			fmt.Printf("Snapshot `%s` was used as a source in following snapshots:\n", snapshot.Name)
 			for _, snap := range snapshots {
@@ -51,7 +48,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = snapshotCollection.Drop(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().Drop(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
@@ -67,7 +64,7 @@ func makeCmdSnapshotDrop() *commander.Command {
 		UsageLine: "drop <name>",
 		Short:     "delete snapshot",
 		Long: `
-Drop removes information about snapshot. If snapshot is published,
+Drop removes information about a snapshot. If snapshot is published,
 it can't be dropped.
 
 Example:

cmd/snapshot_filter.go

@@ -0,0 +1,107 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"sort"
+	"strings"
+)
+
+func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 3 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+
+	// Load <source> snapshot
+	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Convert snapshot to package list
+	context.Progress().Printf("Loading packages (%d)...\n", source.RefList().Len())
+	packageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	context.Progress().Printf("Building indexes...\n")
+	packageList.PrepareIndex()
+
+	// Calculate architectures
+	var architecturesList []string
+
+	if len(context.ArchitecturesList()) > 0 {
+		architecturesList = context.ArchitecturesList()
+	} else {
+		architecturesList = packageList.Architectures(false)
+	}
+
+	sort.Strings(architecturesList)
+
+	if len(architecturesList) == 0 && withDeps {
+		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+	}
+
+	// Initial queries out of arguments
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i, arg := range args[2:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+	}
+
+	// Filter with dependencies as requested
+	result, err := packageList.Filter(queries, withDeps, nil, context.DependencyOptions(), architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Create <destination> snapshot
+	destination := deb.NewSnapshotFromPackageList(args[1], []*deb.Snapshot{source}, result,
+		fmt.Sprintf("Filtered '%s', query was: '%s'", source.Name, strings.Join(args[2:], " ")))
+
+	err = context.CollectionFactory().SnapshotCollection().Add(destination)
+	if err != nil {
+		return fmt.Errorf("unable to create snapshot: %s", err)
+	}
+
+	context.Progress().Printf("\nSnapshot %s successfully filtered.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+
+	return err
+}
+
+func makeCmdSnapshotFilter() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotFilter,
+		UsageLine: "filter <source> <destination> <package-query> ...",
+		Short:     "filter packages in snapshot producing another snapshot",
+		Long: `
+Command filter does filtering in snapshot <source>, producing another
+snapshot <destination>. Packages could be specified simply
+as 'package-name' or as package queries.
+
+Example:
+
+    $ aptly snapshot filter wheezy-main wheezy-required 'Priorioty (required)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-filter", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependent packages as well")
+
+	return cmd
+}

cmd/snapshot_list.go

@@ -2,41 +2,94 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"sort"
 )
 
+// Snapshot sorting methods
+const (
+	SortName = iota
+	SortTime
+)
+
+type snapshotListToSort struct {
+	list       []*deb.Snapshot
+	sortMethod int
+}
+
+func parseSortMethod(sortMethod string) (int, error) {
+	switch sortMethod {
+	case "time", "Time":
+		return SortTime, nil
+	case "name", "Name":
+		return SortName, nil
+	}
+
+	return -1, fmt.Errorf("sorting method \"%s\" unknown", sortMethod)
+}
+
+func (s snapshotListToSort) Swap(i, j int) {
+	s.list[i], s.list[j] = s.list[j], s.list[i]
+}
+
+func (s snapshotListToSort) Less(i, j int) bool {
+	switch s.sortMethod {
+	case SortName:
+		return s.list[i].Name < s.list[j].Name
+	case SortTime:
+		return s.list[i].CreatedAt.Before(s.list[j].CreatedAt)
+	}
+	panic("unknown sort method")
+}
+
+func (s snapshotListToSort) Len() int {
+	return len(s.list)
+}
+
 func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
+
+	snapshotsToSort := &snapshotListToSort{}
+	snapshotsToSort.list = make([]*deb.Snapshot, context.CollectionFactory().SnapshotCollection().Len())
+	snapshotsToSort.sortMethod, err = parseSortMethod(sortMethodString)
+	if err != nil {
 		return err
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	i := 0
+	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		snapshotsToSort.list[i] = snapshot
+		i++
 
-	if snapshotCollection.Len() > 0 {
-		fmt.Printf("List of snapshots:\n")
+		return nil
+	})
 
-		snapshots := make([]string, snapshotCollection.Len())
+	sort.Sort(snapshotsToSort)
 
-		i := 0
-		snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
-			snapshots[i] = snapshot.String()
-			i++
-			return nil
-		})
-
-		sort.Strings(snapshots)
-		for _, snapshot := range snapshots {
-			fmt.Printf(" * %s\n", snapshot)
+	if raw {
+		for _, snapshot := range snapshotsToSort.list {
+			fmt.Printf("%s\n", snapshot.Name)
 		}
-
-		fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
 	} else {
-		fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n")
+		if len(snapshotsToSort.list) > 0 {
+			fmt.Printf("List of snapshots:\n")
+
+			for _, snapshot := range snapshotsToSort.list {
+				fmt.Printf(" * %s\n", snapshot.String())
+			}
+
+			fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
+		} else {
+			fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n")
+		}
 	}
 	return err
 
@@ -54,8 +107,10 @@ Example:
 
   $ aptly snapshot list
 `,
-		Flag: *flag.NewFlagSet("aptly-snapshot-list", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+	cmd.Flag.String("sort", "name", "display list in 'name' or creation 'time' order")
+
 	return cmd
 }

cmd/snapshot_merge.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"strings"
 )
 
@@ -12,29 +11,39 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-
-	sources := make([]*debian.Snapshot, len(args)-1)
+	sources := make([]*deb.Snapshot, len(args)-1)
 
 	for i := 0; i < len(args)-1; i++ {
-		sources[i], err = snapshotCollection.ByName(args[i+1])
+		sources[i], err = context.CollectionFactory().SnapshotCollection().ByName(args[i+1])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
 
-		err = snapshotCollection.LoadComplete(sources[i])
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(sources[i])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
 	}
 
-	result := sources[0].RefList()
+	latest := context.flags.Lookup("latest").Value.Get().(bool)
+    noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
 
+    if noRemove && latest {
+        return fmt.Errorf("-no-remove and -latest can't be specified together")
+    }
+
+	overrideMatching := !latest && !noRemove
+
+	result := sources[0].RefList()
 	for i := 1; i < len(sources); i++ {
-		result = result.Merge(sources[i].RefList(), true)
+		result = result.Merge(sources[i].RefList(), overrideMatching)
+	}
+
+	if latest {
+		deb.FilterLatestRefs(result)
 	}
 
 	sourceDescription := make([]string, len(sources))
@@ -43,10 +52,10 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	}
 
 	// Create <destination> snapshot
-	destination := debian.NewSnapshotFromRefList(args[0], sources, result,
+	destination := deb.NewSnapshotFromRefList(args[0], sources, result,
 		fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
-	err = snapshotCollection.Add(destination)
+	err = context.CollectionFactory().SnapshotCollection().Add(destination)
 	if err != nil {
 		return fmt.Errorf("unable to create snapshot: %s", err)
 	}
@@ -62,18 +71,20 @@ func makeCmdSnapshotMerge() *commander.Command {
 		UsageLine: "merge <destination> <source> [<source>...]",
 		Short:     "merges snapshots",
 		Long: `
-Merge merges several <source> snapshots into one <destination> snapshot.
-Merge happens from left to right. Packages with the same name-architecture
-pair are replaced during merge (package from latest snapshot on the list
-wins). If run with only one source snapshot, merge copies <source> into
+Merge command merges several <source> snapshots into one <destination> snapshot.
+Merge happens from left to right. By default, packages with the same
+name-architecture pair are replaced during merge (package from latest snapshot
+on the list wins).  If run with only one source snapshot, merge copies <source> into
 <destination>.
 
 Example:
 
     $ aptly snapshot merge wheezy-w-backports wheezy-main wheezy-backports
 `,
-		Flag: *flag.NewFlagSet("aptly-snapshot-merge", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("latest", false, "use only the latest version of each package")
+    cmd.Flag.Bool("no-remove", false, "don't remove duplicate arch/name packages")
+
 	return cmd
 }

cmd/snapshot_pull.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"sort"
 	"strings"
 )
@@ -13,61 +14,59 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 4 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	noDeps := cmd.Flag.Lookup("no-deps").Value.Get().(bool)
-	noRemove := cmd.Flag.Lookup("no-remove").Value.Get().(bool)
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	packageCollection := debian.NewPackageCollection(context.database)
+	noDeps := context.flags.Lookup("no-deps").Value.Get().(bool)
+	noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
+	allMatches := context.flags.Lookup("all-matches").Value.Get().(bool)
 
 	// Load <name> snapshot
-	snapshot, err := snapshotCollection.ByName(args[0])
+	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
 	// Load <source> snapshot
-	source, err := snapshotCollection.ByName(args[1])
+	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(source)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	context.progress.Printf("Dependencies would be pulled into snapshot:\n    %s\nfrom snapshot:\n    %s\nand result would be saved as new snapshot %s.\n",
+	context.Progress().Printf("Dependencies would be pulled into snapshot:\n    %s\nfrom snapshot:\n    %s\nand result would be saved as new snapshot %s.\n",
 		snapshot, source, args[2])
 
 	// Convert snapshot to package list
-	context.progress.Printf("Loading packages (%d)...\n", snapshot.RefList().Len()+source.RefList().Len())
-	packageList, err := debian.NewPackageListFromRefList(snapshot.RefList(), packageCollection, context.progress)
+	context.Progress().Printf("Loading packages (%d)...\n", snapshot.RefList().Len()+source.RefList().Len())
+	packageList, err := deb.NewPackageListFromRefList(snapshot.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	sourcePackageList, err := debian.NewPackageListFromRefList(source.RefList(), packageCollection, context.progress)
+	sourcePackageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	context.progress.Printf("Building indexes...\n")
+	context.Progress().Printf("Building indexes...\n")
 	packageList.PrepareIndex()
 	sourcePackageList.PrepareIndex()
 
 	// Calculate architectures
 	var architecturesList []string
 
-	if len(context.architecturesList) > 0 {
-		architecturesList = context.architecturesList
+	if len(context.ArchitecturesList()) > 0 {
+		architecturesList = context.ArchitecturesList()
 	} else {
 		architecturesList = packageList.Architectures(false)
 	}
@@ -78,90 +77,71 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
 	}
 
-	// Initial dependencies out of arguments
-	initialDependencies := make([]debian.Dependency, len(args)-3)
-	for i, arg := range args[3:] {
-		initialDependencies[i], err = debian.ParseDependency(arg)
-		if err != nil {
-			return fmt.Errorf("unable to parse argument: %s", err)
-		}
-	}
-
-	// Perform pull
+	// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
+	var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
 	for _, arch := range architecturesList {
-		dependencies := make([]debian.Dependency, len(initialDependencies), 128)
-		for i := range dependencies {
-			dependencies[i] = initialDependencies[i]
-			dependencies[i].Architecture = arch
-		}
-
-		// Go over list of initial dependencies + list of dependencies found
-		for i := 0; i < len(dependencies); i++ {
-			dep := dependencies[i]
-
-			// Search for package that can satisfy dependencies
-			pkg := sourcePackageList.Search(dep)
-			if pkg == nil {
-				context.progress.ColoredPrintf("@y[!]@| @!Dependency %s can't be satisfied with source %s@|", &dep, source)
-				continue
-			}
-
-			if !noRemove {
-				// Remove all packages with the same name and architecture
-				for p := packageList.Search(debian.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}); p != nil; {
-					packageList.Remove(p)
-					context.progress.ColoredPrintf("@r[-]@| %s removed", p)
-					p = packageList.Search(debian.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name})
-				}
-			}
-
-			// Add new discovered package
-			packageList.Add(pkg)
-			context.progress.ColoredPrintf("@g[+]@| %s added", pkg)
-
-			if noDeps {
-				continue
-			}
-
-			// Find missing dependencies for single added package
-			pL := debian.NewPackageList()
-			pL.Add(pkg)
-
-			missing, err := pL.VerifyDependencies(context.dependencyOptions, []string{arch}, packageList, nil)
-			if err != nil {
-				context.progress.ColoredPrintf("@y[!]@| @!Error while verifying dependencies for pkg %s: %s@|", pkg, err)
-			}
-
-			// Append missing dependencies to the list of dependencies to satisfy
-			for _, misDep := range missing {
-				found := false
-				for _, d := range dependencies {
-					if d == misDep {
-						found = true
-						break
-					}
-				}
-
-				if !found {
-					dependencies = append(dependencies, misDep)
-				}
-			}
-		}
+		archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
 	}
 
-	if cmd.Flag.Lookup("dry-run").Value.Get().(bool) {
-		context.progress.Printf("\nNot creating snapshot, as dry run was requested.\n")
+	// Initial queries out of arguments
+	queries := make([]deb.PackageQuery, len(args)-3)
+	for i, arg := range args[3:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+		// Add architecture filter
+		queries[i] = &deb.AndQuery{queries[i], archQuery}
+	}
+
+	// Filter with dependencies as requested
+	result, err := sourcePackageList.Filter(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+	result.PrepareIndex()
+
+	alreadySeen := map[string]bool{}
+
+	result.ForEachIndexed(func(pkg *deb.Package) error {
+		key := pkg.Architecture + "_" + pkg.Name
+		_, seen := alreadySeen[key]
+
+		// If we haven't seen such name-architecture pair and were instructed to remove, remove it
+		if !noRemove && !seen {
+			// Remove all packages with the same name and architecture
+			pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
+			for _, p := range pS {
+				packageList.Remove(p)
+				context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
+			}
+		}
+
+		// If !allMatches, add only first matching name-arch package
+		if !seen || allMatches {
+			packageList.Add(pkg)
+			context.Progress().ColoredPrintf("@g[+]@| %s added", pkg)
+		}
+
+		alreadySeen[key] = true
+
+		return nil
+	})
+	alreadySeen = nil
+
+	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+		context.Progress().Printf("\nNot creating snapshot, as dry run was requested.\n")
 	} else {
 		// Create <destination> snapshot
-		destination := debian.NewSnapshotFromPackageList(args[2], []*debian.Snapshot{snapshot, source}, packageList,
+		destination := deb.NewSnapshotFromPackageList(args[2], []*deb.Snapshot{snapshot, source}, packageList,
 			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", snapshot.Name, source.Name, strings.Join(args[3:], " ")))
 
-		err = snapshotCollection.Add(destination)
+		err = context.CollectionFactory().SnapshotCollection().Add(destination)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		context.progress.Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+		context.Progress().Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
 	}
 	return err
 }
@@ -169,14 +149,14 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 func makeCmdSnapshotPull() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotPull,
-		UsageLine: "pull <name> <source> <destination> <package-name> ...",
+		UsageLine: "pull <name> <source> <destination> <package-query> ...",
 		Short:     "pull packages from another snapshot",
 		Long: `
-Command pull pulls new packages along with its dependencies to snapshot <name>
+Command pull pulls new packages along with its' dependencies to snapshot <name>
 from snapshot <source>. Pull can upgrade package version in <name> with
 versions from <source> following dependencies. New snapshot <destination>
-is created as result of this process. Packages could be specified simply
-as 'package-name' or as dependency 'package-name (>= version)'.
+is created as a result of this process. Packages could be specified simply
+as 'package-name' or as package queries.
 
 Example:
 
@@ -188,6 +168,7 @@ Example:
 	cmd.Flag.Bool("dry-run", false, "don't create destination snapshot, just show what would be pulled")
 	cmd.Flag.Bool("no-deps", false, "don't process dependencies, just pull listed packages")
 	cmd.Flag.Bool("no-remove", false, "don't remove other package versions when pulling package")
+	cmd.Flag.Bool("all-matches", false, "pull all the packages that satisfy the dependency version requirements")
 
 	return cmd
 }

cmd/snapshot_rename.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlySnapshotRename(cmd *commander.Command, args []string) error {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().SnapshotCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: snapshot %s already exists", newName)
+	}
+
+	snapshot.Name = newName
+	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nSnapshot %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdSnapshotRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames snapshot",
+		Long: `
+Command changes name of the snapshot. Snapshot name should be unique.
+
+Example:
+
+  $ aptly snapshot rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/snapshot_search.go

@@ -0,0 +1,125 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"sort"
+)
+
+func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+	command := cmd.Parent.Name()
+
+	var reflist *deb.PackageRefList
+
+	if command == "snapshot" {
+		snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = snapshot.RefList()
+	} else if command == "mirror" {
+		repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else if command == "repo" {
+		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else {
+		panic("unknown command")
+	}
+
+	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	list.PrepareIndex()
+
+	q, err := query.Parse(args[1])
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+	architecturesList := []string{}
+
+	if withDeps {
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
+		} else {
+			architecturesList = list.Architectures(false)
+		}
+
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+		}
+	}
+
+	result, err := list.Filter([]deb.PackageQuery{q}, withDeps,
+		nil, context.DependencyOptions(), architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	result.ForEach(func(p *deb.Package) error {
+		context.Progress().Printf("%s\n", p)
+		return nil
+	})
+
+	return err
+}
+
+func makeCmdSnapshotSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> <package-query>",
+		Short:     "search snapshot for packages matching query",
+		Long: `
+Command search displays list of packages in snapshot that match package query
+
+Example:
+
+    $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-search", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+
+	return cmd
+}

cmd/snapshot_show.go

@@ -2,27 +2,25 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	snapshot, err := snapshotCollection.ByName(name)
+	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -32,7 +30,7 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Description: %s\n", snapshot.Description)
 	fmt.Printf("Number of packages: %d\n", snapshot.NumPackages())
 
-	withPackages := cmd.Flag.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		ListPackagesRefList(snapshot.RefList())
 	}
@@ -46,7 +44,7 @@ func makeCmdSnapshotShow() *commander.Command {
 		UsageLine: "show <name>",
 		Short:     "shows details about snapshot",
 		Long: `
-Command show displays full information about snapshot.
+Command show displays full information about a snapshot.
 
 Example:
 

cmd/snapshot_verify.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
 	"sort"
 )
 
@@ -12,40 +11,38 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	packageCollection := debian.NewPackageCollection(context.database)
-
-	snapshots := make([]*debian.Snapshot, len(args))
+	snapshots := make([]*deb.Snapshot, len(args))
 	for i := range snapshots {
-		snapshots[i], err = snapshotCollection.ByName(args[i])
+		snapshots[i], err = context.CollectionFactory().SnapshotCollection().ByName(args[i])
 		if err != nil {
 			return fmt.Errorf("unable to verify: %s", err)
 		}
 
-		err = snapshotCollection.LoadComplete(snapshots[i])
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshots[i])
 		if err != nil {
 			return fmt.Errorf("unable to verify: %s", err)
 		}
 	}
 
-	context.progress.Printf("Loading packages...\n")
+	context.Progress().Printf("Loading packages...\n")
 
-	packageList, err := debian.NewPackageListFromRefList(snapshots[0].RefList(), packageCollection, context.progress)
+	packageList, err := deb.NewPackageListFromRefList(snapshots[0].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	sourcePackageList := debian.NewPackageList()
+	sourcePackageList := deb.NewPackageList()
 	err = sourcePackageList.Append(packageList)
 	if err != nil {
 		fmt.Errorf("unable to merge sources: %s", err)
 	}
 
+	var pL *deb.PackageList
 	for i := 1; i < len(snapshots); i++ {
-		pL, err := debian.NewPackageListFromRefList(snapshots[i].RefList(), packageCollection, context.progress)
+		pL, err = deb.NewPackageListFromRefList(snapshots[i].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 		if err != nil {
 			fmt.Errorf("unable to load packages: %s", err)
 		}
@@ -60,8 +57,8 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 
 	var architecturesList []string
 
-	if len(context.architecturesList) > 0 {
-		architecturesList = context.architecturesList
+	if len(context.ArchitecturesList()) > 0 {
+		architecturesList = context.ArchitecturesList()
 	} else {
 		architecturesList = packageList.Architectures(true)
 	}
@@ -70,17 +67,17 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
 	}
 
-	context.progress.Printf("Verifying...\n")
+	context.Progress().Printf("Verifying...\n")
 
-	missing, err := packageList.VerifyDependencies(context.dependencyOptions, architecturesList, sourcePackageList, context.progress)
+	missing, err := packageList.VerifyDependencies(context.DependencyOptions(), architecturesList, sourcePackageList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to verify dependencies: %s", err)
 	}
 
 	if len(missing) == 0 {
-		context.progress.Printf("All dependencies are satisfied.\n")
+		context.Progress().Printf("All dependencies are satisfied.\n")
 	} else {
-		context.progress.Printf("Missing dependencies (%d):\n", len(missing))
+		context.Progress().Printf("Missing dependencies (%d):\n", len(missing))
 		deps := make([]string, len(missing))
 		i := 0
 		for _, dep := range missing {
@@ -91,7 +88,7 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 		sort.Strings(deps)
 
 		for _, dep := range deps {
-			context.progress.Printf("  %s\n", dep)
+			context.Progress().Printf("  %s\n", dep)
 		}
 	}
 
@@ -104,7 +101,7 @@ func makeCmdSnapshotVerify() *commander.Command {
 		UsageLine: "verify <name> [<source> ...]",
 		Short:     "verify dependencies in snapshot",
 		Long: `
-Verify does depenency resolution in snapshot <name>, possibly using additional
+Verify does dependency resolution in snapshot <name>, possibly using additional
 snapshots <source> as dependency sources. All unsatisfied dependencies are
 printed.
 
@@ -112,7 +109,6 @@ Example:
 
     $ aptly snapshot verify wheezy-main wheezy-contrib wheezy-non-free
 `,
-		Flag: *flag.NewFlagSet("aptly-snapshot-verify", flag.ExitOnError),
 	}
 
 	return cmd

cmd/task.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdTask() *commander.Command {
+	return &commander.Command{
+		UsageLine: "task",
+		Short:     "manage aptly tasks",
+		Subcommands: []*commander.Command{
+			makeCmdTaskRun(),
+		},
+	}
+}

cmd/task_run.go

@@ -0,0 +1,148 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mattn/go-shellwords"
+	"github.com/smira/commander"
+)
+
+func aptlyTaskRun(cmd *commander.Command, args []string) error {
+	var err error
+	var cmdList [][]string
+
+	if filename := cmd.Flag.Lookup("filename").Value.Get().(string); filename != "" {
+		var text string
+		cmdArgs := []string{}
+
+		if finfo, err := os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
+			return fmt.Errorf("no such file, %s\n", filename)
+		}
+
+		fmt.Println("Reading file...\n")
+
+		file, err := os.Open(filename)
+
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		scanner := bufio.NewScanner(file)
+
+		for scanner.Scan() {
+			text = strings.TrimSpace(scanner.Text()) + ","
+			parsedArgs, _ := shellwords.Parse(text)
+			cmdArgs = append(cmdArgs, parsedArgs...)
+		}
+
+		if err = scanner.Err(); err != nil {
+			return err
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("the file is empty")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else if len(args) == 0 {
+		var text string
+		cmdArgs := []string{}
+
+		fmt.Println("Please enter one command per line and leave one blank when finished.")
+
+		reader := bufio.NewReader(os.Stdin)
+		for {
+			fmt.Printf("> ")
+			text, _ = reader.ReadString('\n')
+			if text == "\n" {
+				break
+			} else {
+				text = strings.TrimSpace(text) + ","
+				parsedArgs, _ := shellwords.Parse(text)
+				cmdArgs = append(cmdArgs, parsedArgs...)
+			}
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("nothing entered")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else {
+		cmdList = formatCommands(args)
+	}
+
+	commandErrored := false
+
+	for i, command := range cmdList {
+		if !commandErrored {
+			context.Progress().ColoredPrintf("@g%d) [Running]: %s@!", (i + 1), strings.Join(command, " "))
+			context.Progress().ColoredPrintf("\n@yBegin command output: ----------------------------@!")
+			context.Progress().Flush()
+
+			returnCode := Run(RootCommand(), command, false)
+			if returnCode != 0 {
+				commandErrored = true
+			}
+			context.Progress().ColoredPrintf("\n@yEnd command output: ------------------------------@!")
+			CleanupContext()
+		} else {
+			context.Progress().ColoredPrintf("@r%d) [Skipping]: %s@!", (i + 1), strings.Join(command, " "))
+		}
+	}
+
+	if commandErrored {
+		err = fmt.Errorf("at least one command has reported an error")
+	}
+
+	return err
+}
+
+func formatCommands(args []string) [][]string {
+	var cmd []string
+	var cmdArray [][]string
+
+	for _, s := range args {
+		if sTrimmed := strings.TrimRight(s, ","); sTrimmed != s {
+			cmd = append(cmd, sTrimmed)
+			cmdArray = append(cmdArray, cmd)
+			cmd = []string{}
+		} else {
+			cmd = append(cmd, s)
+		}
+	}
+
+	if len(cmd) > 0 {
+		cmdArray = append(cmdArray, cmd)
+	}
+
+	return cmdArray
+}
+
+func makeCmdTaskRun() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyTaskRun,
+		UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
+		Short:     "run aptly tasks",
+		Long: `
+Command helps origanise multiple aptly commands in one single aptly task, running as single thread.
+
+Example:
+
+  $ aptly task run
+  > repo create local
+  > repo add local pkg1
+  > publish repo local
+  > serve
+  >
+
+`,
+	}
+
+	cmd.Flag.String("filename", "", "specifies the filename that contains the commands to run")
+	return cmd
+}

cmd/version.go

@@ -2,12 +2,16 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/commander"
 )
 
 func aptlyVersion(cmd *commander.Command, args []string) error {
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
 	fmt.Printf("aptly version: %s\n", aptly.Version)
 	return nil
 }
@@ -23,6 +27,5 @@ Shows aptly version.
 ex:
   $ aptly version
 `,
-		Flag: *flag.NewFlagSet("aptly-version", flag.ExitOnError),
 	}
 }

database/leveldb.go

@@ -7,6 +7,8 @@ import (
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/filter"
 	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 // Errors for Storage
@@ -22,11 +24,14 @@ type Storage interface {
 	KeysByPrefix(prefix []byte) [][]byte
 	FetchByPrefix(prefix []byte) [][]byte
 	Close() error
+	ReOpen() error
 	StartBatch()
 	FinishBatch() error
+	CompactDB() error
 }
 
 type levelDB struct {
+	path  string
 	db    *leveldb.DB
 	batch *leveldb.Batch
 }
@@ -36,19 +41,42 @@ var (
 	_ Storage = &levelDB{}
 )
 
-// OpenDB opens (creates) LevelDB database
-func OpenDB(path string) (Storage, error) {
+func internalOpen(path string) (*leveldb.DB, error) {
 	o := &opt.Options{
 		Filter: filter.NewBloomFilter(10),
 	}
 
-	db, err := leveldb.OpenFile(path, o)
+	return leveldb.OpenFile(path, o)
+}
+
+// OpenDB opens (creates) LevelDB database
+func OpenDB(path string) (Storage, error) {
+	db, err := internalOpen(path)
 	if err != nil {
 		return nil, err
 	}
-	return &levelDB{db: db}, nil
+	return &levelDB{db: db, path: path}, nil
 }
 
+// RecoverDB recovers LevelDB database from corruption
+func RecoverDB(path string) error {
+	stor, err := storage.OpenFile(path)
+	if err != nil {
+		return err
+	}
+
+	db, err := leveldb.Recover(stor, nil)
+	if err != nil {
+		return err
+	}
+
+	db.Close()
+	stor.Close()
+
+	return nil
+}
+
+// Get key value from database
 func (l *levelDB) Get(key []byte) ([]byte, error) {
 	value, err := l.db.Get(key, nil)
 	if err != nil {
@@ -61,6 +89,7 @@ func (l *levelDB) Get(key []byte) ([]byte, error) {
 	return value, nil
 }
 
+// Put saves key to database, if key has the same value in DB already, it is not saved
 func (l *levelDB) Put(key []byte, value []byte) error {
 	if l.batch != nil {
 		l.batch.Put(key, value)
@@ -79,6 +108,7 @@ func (l *levelDB) Put(key []byte, value []byte) error {
 	return l.db.Put(key, value, nil)
 }
 
+// Delete removes key from DB
 func (l *levelDB) Delete(key []byte) error {
 	if l.batch != nil {
 		l.batch.Delete(key)
@@ -87,6 +117,7 @@ func (l *levelDB) Delete(key []byte) error {
 	return l.db.Delete(key, nil)
 }
 
+// KeysByPrefix returns all keys that start with prefix
 func (l *levelDB) KeysByPrefix(prefix []byte) [][]byte {
 	result := make([][]byte, 0, 20)
 
@@ -103,6 +134,7 @@ func (l *levelDB) KeysByPrefix(prefix []byte) [][]byte {
 	return result
 }
 
+// FetchByPrefix returns all values with keys that start with prefix
 func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
 	result := make([][]byte, 0, 20)
 
@@ -119,10 +151,30 @@ func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
 	return result
 }
 
+// Close finishes DB work
 func (l *levelDB) Close() error {
-	return l.db.Close()
+	if l.db == nil {
+		return nil
+	}
+	err := l.db.Close()
+	l.db = nil
+	return err
 }
 
+// Reopen tries to re-open the database
+func (l *levelDB) ReOpen() error {
+	if l.db != nil {
+		return nil
+	}
+
+	var err error
+	l.db, err = internalOpen(l.path)
+	return err
+}
+
+// StartBatch starts batch processing of keys
+//
+// All subsequent Get, Put and Delete would work on batch
 func (l *levelDB) StartBatch() {
 	if l.batch != nil {
 		panic("batch already started")
@@ -130,6 +182,7 @@ func (l *levelDB) StartBatch() {
 	l.batch = new(leveldb.Batch)
 }
 
+// FinishBatch finalizes the batch, saving operations
 func (l *levelDB) FinishBatch() error {
 	if l.batch == nil {
 		panic("no batch")
@@ -138,3 +191,8 @@ func (l *levelDB) FinishBatch() error {
 	l.batch = nil
 	return err
 }
+
+// CompactDB compacts database by merging layers
+func (l *levelDB) CompactDB() error {
+	return l.db.CompactRange(util.Range{})
+}

database/leveldb_test.go

@@ -11,7 +11,8 @@ func Test(t *testing.T) {
 }
 
 type LevelDBSuite struct {
-	db Storage
+	path string
+	db   Storage
 }
 
 var _ = Suite(&LevelDBSuite{})
@@ -19,7 +20,8 @@ var _ = Suite(&LevelDBSuite{})
 func (s *LevelDBSuite) SetUpTest(c *C) {
 	var err error
 
-	s.db, err = OpenDB(c.MkDir())
+	s.path = c.MkDir()
+	s.db, err = OpenDB(s.path)
 	c.Assert(err, IsNil)
 }
 
@@ -28,6 +30,29 @@ func (s *LevelDBSuite) TearDownTest(c *C) {
 	c.Assert(err, IsNil)
 }
 
+func (s *LevelDBSuite) TestRecoverDB(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Check(err, IsNil)
+
+	err = s.db.Close()
+	c.Check(err, IsNil)
+
+	err = RecoverDB(s.path)
+	c.Check(err, IsNil)
+
+	s.db, err = OpenDB(s.path)
+	c.Check(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}
+
 func (s *LevelDBSuite) TestGetPut(c *C) {
 	var (
 		key   = []byte("key")
@@ -122,3 +147,31 @@ func (s *LevelDBSuite) TestBatch(c *C) {
 	s.db.StartBatch()
 	c.Check(func() { s.db.StartBatch() }, Panics, "batch already started")
 }
+
+func (s *LevelDBSuite) TestCompactDB(c *C) {
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+
+	c.Check(s.db.CompactDB(), IsNil)
+}
+
+func (s *LevelDBSuite) TestReOpen(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	err = s.db.Close()
+	c.Assert(err, IsNil)
+
+	err = s.db.ReOpen()
+	c.Assert(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}

deb/collections.go

@@ -0,0 +1,65 @@
+package deb
+
+import (
+	"github.com/smira/aptly/database"
+)
+
+// CollectionFactory is a single place to generate all desired collections
+type CollectionFactory struct {
+	db             database.Storage
+	packages       *PackageCollection
+	remoteRepos    *RemoteRepoCollection
+	snapshots      *SnapshotCollection
+	localRepos     *LocalRepoCollection
+	publishedRepos *PublishedRepoCollection
+}
+
+// NewCollectionFactory creates new factory
+func NewCollectionFactory(db database.Storage) *CollectionFactory {
+	return &CollectionFactory{db: db}
+}
+
+// PackageCollection returns (or creates) new PackageCollection
+func (factory *CollectionFactory) PackageCollection() *PackageCollection {
+	if factory.packages == nil {
+		factory.packages = NewPackageCollection(factory.db)
+	}
+
+	return factory.packages
+}
+
+// RemoteRepoCollection returns (or creates) new RemoteRepoCollection
+func (factory *CollectionFactory) RemoteRepoCollection() *RemoteRepoCollection {
+	if factory.remoteRepos == nil {
+		factory.remoteRepos = NewRemoteRepoCollection(factory.db)
+	}
+
+	return factory.remoteRepos
+}
+
+// SnapshotCollection returns (or creates) new SnapshotCollection
+func (factory *CollectionFactory) SnapshotCollection() *SnapshotCollection {
+	if factory.snapshots == nil {
+		factory.snapshots = NewSnapshotCollection(factory.db)
+	}
+
+	return factory.snapshots
+}
+
+// LocalRepoCollection returns (or creates) new LocalRepoCollection
+func (factory *CollectionFactory) LocalRepoCollection() *LocalRepoCollection {
+	if factory.localRepos == nil {
+		factory.localRepos = NewLocalRepoCollection(factory.db)
+	}
+
+	return factory.localRepos
+}
+
+// PublishedRepoCollection returns (or creates) new PublishedRepoCollection
+func (factory *CollectionFactory) PublishedRepoCollection() *PublishedRepoCollection {
+	if factory.publishedRepos == nil {
+		factory.publishedRepos = NewPublishedRepoCollection(factory.db)
+	}
+
+	return factory.publishedRepos
+}

deb/deb.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"archive/tar"

deb/deb_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"github.com/smira/aptly/utils"

deb/debian.go

@@ -0,0 +1,2 @@
+// Package deb implements Debian-specific repository handling
+package deb

deb/debian_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	. "launchpad.net/gocheck"

deb/format.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bufio"
@@ -11,8 +11,9 @@ import (
 type Stanza map[string]string
 
 // Canonical order of fields in stanza
-var canocialOrder = []string{"Origin", "Label", "Suite", "Package", "Version", "Installed-Size", "Priority", "Section", "Maintainer",
-	"Architecture", "Codename", "Date", "Architectures", "Components", "Description", "MD5sum", "MD5Sum", "SHA1", "SHA256"}
+var canocialOrder = []string{"Package", "Origin", "Label", "Suite", "Version", "Installed-Size", "Priority", "Section", "Maintainer",
+	"Architecture", "Codename", "Date", "Architectures", "Components", "Description", "MD5sum", "MD5Sum", "SHA1", "SHA256",
+	"Archive", "Component"}
 
 // Copy returns copy of Stanza
 func (s Stanza) Copy() (result Stanza) {

deb/format_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bufio"

deb/index_files.go

@@ -0,0 +1,254 @@
+package deb
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type indexFiles struct {
+	publishedStorage aptly.PublishedStorage
+	basePath         string
+	renameMap        map[string]string
+	generatedFiles   map[string]utils.ChecksumInfo
+	tempDir          string
+	suffix           string
+	indexes          map[string]*indexFile
+}
+
+type indexFile struct {
+	parent       *indexFiles
+	discardable  bool
+	compressable bool
+	signable     bool
+	relativePath string
+	tempFilename string
+	tempFile     *os.File
+	w            *bufio.Writer
+}
+
+func (file *indexFile) BufWriter() (*bufio.Writer, error) {
+	if file.w == nil {
+		var err error
+		file.tempFilename = filepath.Join(file.parent.tempDir, strings.Replace(file.relativePath, "/", "_", -1))
+		file.tempFile, err = os.Create(file.tempFilename)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create temporary index file: %s", err)
+		}
+
+		file.w = bufio.NewWriter(file.tempFile)
+	}
+
+	return file.w, nil
+}
+
+func (file *indexFile) Finalize(signer utils.Signer) error {
+	if file.w == nil {
+		if file.discardable {
+			return nil
+		}
+		file.BufWriter()
+	}
+
+	err := file.w.Flush()
+	if err != nil {
+		file.tempFile.Close()
+		return fmt.Errorf("unable to write to index file: %s", err)
+	}
+
+	if file.compressable {
+		err = utils.CompressFile(file.tempFile)
+		if err != nil {
+			file.tempFile.Close()
+			return fmt.Errorf("unable to compress index file: %s", err)
+		}
+	}
+
+	file.tempFile.Close()
+
+	exts := []string{""}
+	if file.compressable {
+		exts = append(exts, ".gz", ".bz2")
+	}
+
+	for _, ext := range exts {
+		var checksumInfo utils.ChecksumInfo
+
+		checksumInfo, err = utils.ChecksumsForFile(file.tempFilename + ext)
+		if err != nil {
+			return fmt.Errorf("unable to collect checksums: %s", err)
+		}
+		file.parent.generatedFiles[file.relativePath+ext] = checksumInfo
+	}
+
+	err = file.parent.publishedStorage.MkDir(filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)))
+	if err != nil {
+		return fmt.Errorf("unable to create dir: %s", err)
+	}
+
+	for _, ext := range exts {
+		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext),
+			file.tempFilename+ext)
+		if err != nil {
+			return fmt.Errorf("unable to publish file: %s", err)
+		}
+
+		if file.parent.suffix != "" {
+			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] =
+				filepath.Join(file.parent.basePath, file.relativePath+ext)
+		}
+	}
+
+	if file.signable && signer != nil {
+		err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
+		if err != nil {
+			return fmt.Errorf("unable to detached sign file: %s", err)
+		}
+
+		err = signer.ClearSign(file.tempFilename, filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
+		if err != nil {
+			return fmt.Errorf("unable to clearsign file: %s", err)
+		}
+
+		if file.parent.suffix != "" {
+			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
+				filepath.Join(file.parent.basePath, file.relativePath+".gpg")
+			file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] =
+				filepath.Join(file.parent.basePath, "In"+file.relativePath)
+		}
+
+		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
+			file.tempFilename+".gpg")
+		if err != nil {
+			return fmt.Errorf("unable to publish file: %s", err)
+		}
+
+		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix),
+			filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
+		if err != nil {
+			return fmt.Errorf("unable to publish file: %s", err)
+		}
+	}
+
+	return nil
+}
+
+func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string) *indexFiles {
+	return &indexFiles{
+		publishedStorage: publishedStorage,
+		basePath:         basePath,
+		renameMap:        make(map[string]string),
+		generatedFiles:   make(map[string]utils.ChecksumInfo),
+		tempDir:          tempDir,
+		suffix:           suffix,
+		indexes:          make(map[string]*indexFile),
+	}
+}
+
+func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
+	key := fmt.Sprintf("pi-%s-%s-%s", component, arch, udeb)
+	file, ok := files.indexes[key]
+	if !ok {
+		var relativePath string
+
+		if arch == "source" {
+			relativePath = filepath.Join(component, "source", "Sources")
+		} else {
+			if udeb {
+				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
+			} else {
+				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
+			}
+		}
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  false,
+			compressable: true,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
+func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexFile {
+	key := fmt.Sprintf("ri-%s-%s-%s", component, arch, udeb)
+	file, ok := files.indexes[key]
+	if !ok {
+		var relativePath string
+
+		if arch == "source" {
+			relativePath = filepath.Join(component, "source", "Release")
+		} else {
+			if udeb {
+				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Release")
+			} else {
+				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Release")
+			}
+		}
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  udeb,
+			compressable: false,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
+func (files *indexFiles) ReleaseFile() *indexFile {
+	return &indexFile{
+		parent:       files,
+		discardable:  false,
+		compressable: false,
+		signable:     true,
+		relativePath: "Release",
+	}
+}
+
+func (files *indexFiles) FinalizeAll(progress aptly.Progress) (err error) {
+	if progress != nil {
+		progress.InitBar(int64(len(files.indexes)), false)
+		defer progress.ShutdownBar()
+	}
+
+	for _, file := range files.indexes {
+		err = file.Finalize(nil)
+		if err != nil {
+			return
+		}
+		if progress != nil {
+			progress.AddBar(1)
+		}
+	}
+
+	files.indexes = make(map[string]*indexFile)
+
+	return
+}
+
+func (files *indexFiles) RenameFiles() error {
+	var err error
+
+	for oldName, newName := range files.renameMap {
+		err = files.publishedStorage.RenameFile(oldName, newName)
+		if err != nil {
+			return fmt.Errorf("unable to rename: %s", err)
+		}
+	}
+
+	return nil
+}

deb/list.go

@@ -1,11 +1,10 @@
-package debian
+package deb
 
 import (
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
 	"sort"
-	"strings"
 )
 
 // Dependency options
@@ -42,6 +41,7 @@ type PackageList struct {
 // Verify interface
 var (
 	_ sort.Interface = &PackageList{}
+	_ PackageCatalog = &PackageList{}
 )
 
 // NewPackageList creates empty package list
@@ -63,9 +63,9 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 	}
 
 	err := reflist.ForEach(func(key []byte) error {
-		p, err := collection.ByKey(key)
-		if err != nil {
-			return fmt.Errorf("unable to load package with key %s: %s", key, err)
+		p, err2 := collection.ByKey(key)
+		if err2 != nil {
+			return fmt.Errorf("unable to load package with key %s: %s", key, err2)
 		}
 		if progress != nil {
 			progress.AddBar(1)
@@ -86,11 +86,11 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 
 // Add appends package to package list, additionally checking for uniqueness
 func (l *PackageList) Add(p *Package) error {
-	key := string(p.Key(""))
+	key := string(p.ShortKey(""))
 	existing, ok := l.packages[key]
 	if ok {
 		if !existing.Equals(p) {
-			return fmt.Errorf("conflict in package %s: %#v != %#v", p, existing, p)
+			return fmt.Errorf("conflict in package %s", p)
 		}
 		return nil
 	}
@@ -101,7 +101,7 @@ func (l *PackageList) Add(p *Package) error {
 			l.providesIndex[provides] = append(l.providesIndex[provides], p)
 		}
 
-		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= p.Name })
+		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.lessPackages(p, l.packagesIndex[j]) })
 
 		// insert p into l.packagesIndex in position i
 		l.packagesIndex = append(l.packagesIndex, nil)
@@ -123,6 +123,22 @@ func (l *PackageList) ForEach(handler func(*Package) error) error {
 	return err
 }
 
+// ForEachIndexed calls handler for each package in list in indexed order
+func (l *PackageList) ForEachIndexed(handler func(*Package) error) error {
+	if !l.indexed {
+		panic("list not indexed, can't iterate")
+	}
+
+	var err error
+	for _, p := range l.packagesIndex {
+		err = handler(p)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
 // Len returns number of packages in the list
 func (l *PackageList) Len() int {
 	return len(l.packages)
@@ -137,7 +153,7 @@ func (l *PackageList) Append(pl *PackageList) error {
 		existing, ok := l.packages[k]
 		if ok {
 			if !existing.Equals(p) {
-				return fmt.Errorf("conflict in package %s: %#v != %#v", p, existing, p)
+				return fmt.Errorf("conflict in package %s", p)
 			}
 		} else {
 			l.packages[k] = p
@@ -149,7 +165,7 @@ func (l *PackageList) Append(pl *PackageList) error {
 
 // Remove removes package from the list, and updates index when required
 func (l *PackageList) Remove(p *Package) {
-	delete(l.packages, string(p.Key("")))
+	delete(l.packages, string(p.ShortKey("")))
 	if l.indexed {
 		for _, provides := range p.Provides {
 			for i, pkg := range l.providesIndex[provides] {
@@ -220,6 +236,7 @@ func depSliceDeduplicate(s []Dependency) []Dependency {
 //
 // Analysis would be peformed for each architecture, in specified sources
 func (l *PackageList) VerifyDependencies(options int, architectures []string, sources *PackageList, progress aptly.Progress) ([]Dependency, error) {
+	l.PrepareIndex()
 	missing := make([]Dependency, 0, 128)
 
 	if progress != nil {
@@ -229,7 +246,7 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 	for _, arch := range architectures {
 		cache := make(map[string]bool, 2048)
 
-		for _, p := range l.packages {
+		for _, p := range l.packagesIndex {
 			if progress != nil {
 				progress.AddBar(1)
 			}
@@ -247,7 +264,6 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 				variants = depSliceDeduplicate(variants)
 
 				variantsMissing := make([]Dependency, 0, len(variants))
-				missingCount := 0
 
 				for _, dep := range variants {
 					if dep.Architecture == "" {
@@ -255,35 +271,23 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 					}
 
 					hash := dep.Hash()
-					r, ok := cache[hash]
-					if ok {
-						if !r {
-							missingCount++
-						}
-						continue
+					satisfied, ok := cache[hash]
+					if !ok {
+						satisfied = sources.Search(dep, false) != nil
+						cache[hash] = satisfied
 					}
 
-					if sources.Search(dep) == nil {
+					if !satisfied && !ok {
 						variantsMissing = append(variantsMissing, dep)
-						missingCount++
-					} else {
-						cache[hash] = true
+					}
+
+					if satisfied && options&DepFollowAllVariants == 0 {
+						variantsMissing = nil
+						break
 					}
 				}
 
-				if options&DepFollowAllVariants == DepFollowAllVariants {
-					missing = append(missing, variantsMissing...)
-					for _, dep := range variantsMissing {
-						cache[dep.Hash()] = false
-					}
-				} else {
-					if missingCount == len(variants) {
-						missing = append(missing, variantsMissing...)
-						for _, dep := range variantsMissing {
-							cache[dep.Hash()] = false
-						}
-					}
-				}
+				missing = append(missing, variantsMissing...)
 			}
 		}
 	}
@@ -300,13 +304,29 @@ func (l *PackageList) Swap(i, j int) {
 	l.packagesIndex[i], l.packagesIndex[j] = l.packagesIndex[j], l.packagesIndex[i]
 }
 
-// Compare compares two names in lexographical order
+func (l *PackageList) lessPackages(iPkg, jPkg *Package) bool {
+	if iPkg.Name == jPkg.Name {
+		cmp := CompareVersions(iPkg.Version, jPkg.Version)
+		if cmp == 0 {
+			return iPkg.Architecture < jPkg.Architecture
+		}
+		return cmp == 1
+	}
+
+	return iPkg.Name < jPkg.Name
+}
+
+// Less compares two packages by name (lexographical) and version (latest to oldest)
 func (l *PackageList) Less(i, j int) bool {
-	return l.packagesIndex[i].Name < l.packagesIndex[j].Name
+	return l.lessPackages(l.packagesIndex[i], l.packagesIndex[j])
 }
 
 // PrepareIndex prepares list for indexing
 func (l *PackageList) PrepareIndex() {
+	if l.indexed {
+		return
+	}
+
 	l.packagesIndex = make([]*Package, l.Len())
 	l.providesIndex = make(map[string][]*Package, 128)
 
@@ -325,16 +345,49 @@ func (l *PackageList) PrepareIndex() {
 	l.indexed = true
 }
 
-// Search searches package index for specified package
-func (l *PackageList) Search(dep Dependency) *Package {
+// Scan searches package index using full scan
+func (l *PackageList) Scan(q PackageQuery) (result *PackageList) {
+	result = NewPackageList()
+	for _, pkg := range l.packages {
+		if q.Matches(pkg) {
+			result.Add(pkg)
+		}
+	}
+
+	return
+}
+
+// SearchSupported returns true for PackageList
+func (l *PackageList) SearchSupported() bool {
+	return true
+}
+
+// SearchByKey looks up package by exact key reference
+func (l *PackageList) SearchByKey(arch, name, version string) (result *PackageList) {
+	result = NewPackageList()
+
+	pkg := l.packages["P"+arch+" "+name+" "+version]
+	if pkg != nil {
+		result.Add(pkg)
+	}
+
+	return
+}
+
+// Search searches package index for specified package(s) using optimized queries
+func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*Package) {
 	if !l.indexed {
 		panic("list not indexed, can't search")
 	}
 
 	if dep.Relation == VersionDontCare {
 		for _, p := range l.providesIndex[dep.Pkg] {
-			if p.MatchesArchitecture(dep.Architecture) {
-				return p
+			if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
+				searchResults = append(searchResults, p)
+
+				if !allMatches {
+					break
+				}
 			}
 		}
 	}
@@ -344,16 +397,21 @@ func (l *PackageList) Search(dep Dependency) *Package {
 	for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
 		p := l.packagesIndex[i]
 		if p.MatchesDependency(dep) {
-			return p
+			searchResults = append(searchResults, p)
+
+			if !allMatches {
+				break
+			}
 		}
 
 		i++
 	}
-	return nil
+
+	return
 }
 
 // Filter filters package index by specified queries (ORed together), possibly pulling dependencies
-func (l *PackageList) Filter(queries []string, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
+func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
 	if !l.indexed {
 		panic("list not indexed, can't filter")
 	}
@@ -361,53 +419,17 @@ func (l *PackageList) Filter(queries []string, withDependencies bool, source *Pa
 	result := NewPackageList()
 
 	for _, query := range queries {
-		isDepQuery := strings.IndexAny(query, " (){}=<>") != -1
-
-		if !isDepQuery {
-			// try to interpret query as package string representation
-
-			// convert Package.String() to Package.Key()
-			i := strings.Index(query, "_")
-			if i != -1 {
-				pkg, query := query[:i], query[i+1:]
-				j := strings.LastIndex(query, "_")
-				if j != -1 {
-					version, arch := query[:j], query[j+1:]
-					p := l.packages["P"+arch+" "+pkg+" "+version]
-					if p != nil {
-						result.Add(p)
-						continue
-					}
-				}
-			}
-		}
-
-		// try as dependency
-		dep, err := ParseDependency(query)
-		if err != nil {
-			if isDepQuery {
-				return nil, err
-			}
-			// parsing failed, but probably that wasn't a dep query
-			continue
-		}
-
-		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= dep.Pkg })
-
-		for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
-			p := l.packagesIndex[i]
-			if p.MatchesDependency(dep) {
-				result.Add(p)
-			}
-			i++
-		}
+		result.Append(query.Query(l))
 	}
 
 	if withDependencies {
 		added := result.Len()
+		result.PrepareIndex()
 
 		dependencySource := NewPackageList()
-		dependencySource.Append(source)
+		if source != nil {
+			dependencySource.Append(source)
+		}
 		dependencySource.Append(result)
 		dependencySource.PrepareIndex()
 
@@ -423,11 +445,22 @@ func (l *PackageList) Filter(queries []string, withDependencies bool, source *Pa
 
 			// try to satisfy dependencies
 			for _, dep := range missing {
-				p := l.Search(dep)
-				if p != nil {
-					result.Add(p)
-					dependencySource.Add(p)
-					added++
+				// dependency might have already been satisfied
+				// with packages already been added
+				if result.Search(dep, false) != nil {
+					continue
+				}
+
+				searchResults := l.Search(dep, false)
+				if searchResults != nil {
+					for _, p := range searchResults {
+						result.Add(p)
+						dependencySource.Add(p)
+						added++
+						if dependencyOptions&DepFollowAllVariants == 0 {
+							break
+						}
+					}
 				}
 			}
 		}

deb/list_test.go

@@ -1,12 +1,49 @@
-package debian
+package deb
 
 import (
 	"errors"
 	. "launchpad.net/gocheck"
+	"regexp"
 	"sort"
 	"strings"
 )
 
+type containsChecker struct {
+	*CheckerInfo
+}
+
+func (c *containsChecker) Check(params []interface{}, names []string) (result bool, error string) {
+	var (
+		pkgSlice1 []*Package
+		pkgSlice2 []*Package
+		ok        bool
+	)
+
+	pkgMap := make(map[*Package]bool)
+
+	pkgSlice1, ok = params[0].([]*Package)
+	if !ok {
+		return false, "The first parameter is not a Package slice"
+	}
+	pkgSlice2, ok = params[1].([]*Package)
+	if !ok {
+		return false, "The second parameter is not a Package slice"
+	}
+
+	for _, pkg := range pkgSlice2 {
+		pkgMap[pkg] = true
+	}
+
+	for _, pkg := range pkgSlice1 {
+		if _, ok := pkgMap[pkg]; !ok {
+			return false, ""
+		}
+	}
+	return true, ""
+}
+
+var Contains = &containsChecker{&CheckerInfo{Name: "Contains", Params: []string{"Container", "Expected to contain"}}}
+
 type PackageListSuite struct {
 	// Simple list with "real" packages from stanzas
 	list                   *PackageList
@@ -14,8 +51,10 @@ type PackageListSuite struct {
 
 	// Mocked packages in list
 	packages       []*Package
+	packages2      []*Package
 	sourcePackages []*Package
 	il             *PackageList
+	il2            *PackageList
 }
 
 var _ = Suite(&PackageListSuite{})
@@ -29,7 +68,7 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 	stanza["Package"] = "mars-invaders"
 	s.p3 = NewPackageFromControlFile(stanza)
 	stanza = packageStanza.Copy()
-	stanza["Size"] = "42"
+	stanza["Source"] = "unknown-planet"
 	s.p4 = NewPackageFromControlFile(stanza)
 	stanza = packageStanza.Copy()
 	stanza["Package"] = "lonely-strangers"
@@ -60,6 +99,20 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 	}
 	s.il.PrepareIndex()
 
+	s.il2 = NewPackageList()
+	s.packages2 = []*Package{
+		&Package{Name: "mailer", Version: "3.5.8", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		&Package{Name: "sendmail", Version: "1.0", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		&Package{Name: "app", Version: "1.1-bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		&Package{Name: "app", Version: "1.1-bp2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		&Package{Name: "app", Version: "1.2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
+		&Package{Name: "app", Version: "3.0", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+	}
+	for _, p := range s.packages2 {
+		s.il2.Add(p)
+	}
+	s.il2.PrepareIndex()
+
 	s.sourcePackages = []*Package{
 		&Package{Name: "postfix", Version: "1.3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
 		&Package{Name: "app", Version: "1.1~bp1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
@@ -196,35 +249,60 @@ func (s *PackageListSuite) TestAppend(c *C) {
 }
 
 func (s *PackageListSuite) TestSearch(c *C) {
-	c.Check(func() { s.list.Search(Dependency{Architecture: "i386", Pkg: "app"}) }, Panics, "list not indexed, can't search")
+	//allMatches = False
+	c.Check(func() { s.list.Search(Dependency{Architecture: "i386", Pkg: "app"}, false) }, Panics, "list not indexed, can't search")
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "mail-agent"}), Equals, s.packages[4])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "puppy"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "mail-agent"}, false), DeepEquals, []*Package{s.packages[4]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "puppy"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp2"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp2"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1~~"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1~~"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~bp1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~~"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~~"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.0"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.2"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.0"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.2"}, false), IsNil)
 
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}), Equals, s.packages[3])
-	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}), IsNil)
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}, false), IsNil)
+
+	// search w/o version should return package with latest version
+	c.Check(s.il.Search(Dependency{Architecture: "source", Pkg: "dpkg"}, false), DeepEquals, []*Package{s.packages[13]})
+
+	// allMatches = True
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "mail-agent"}, true), Contains, []*Package{s.packages2[0], s.packages2[1]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "puppy"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "3"}, true), Contains, []*Package{s.packages2[5]})
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLess, Version: "1.2"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLess, Version: "1.1~"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.2"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1-bp1"}, true), Contains, []*Package{s.packages2[2]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.0"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreater, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreater, Version: "5.0"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}, true), Contains, []*Package{s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "5.0"}, true), IsNil)
 }
 
 func (s *PackageListSuite) TestFilter(c *C) {
-	c.Check(func() { s.list.Filter([]string{"abcd_0.3_i386"}, false, nil, 0, nil) }, Panics, "list not indexed, can't filter")
-
-	_, err := s.il.Filter([]string{"app >3)"}, false, nil, 0, nil)
-	c.Check(err, ErrorMatches, "unable to parse dependency.*")
+	c.Check(func() { s.list.Filter([]PackageQuery{&PkgQuery{"abcd", "0.3", "i386"}}, false, nil, 0, nil) }, Panics, "list not indexed, can't filter")
 
 	plString := func(l *PackageList) string {
 		list := make([]string, 0, l.Len())
@@ -237,25 +315,69 @@ func (s *PackageListSuite) TestFilter(c *C) {
 		return strings.Join(list, " ")
 	}
 
-	result, err := s.il.Filter([]string{"app_1.1~bp1_i386"}, false, nil, 0, nil)
+	result, err := s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
 
-	result, err = s.il.Filter([]string{"app_1.1~bp1_i386", "dpkg_1.7_source", "dpkg_1.8_amd64"}, false, nil, 0, nil)
+	result, err = s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, &PkgQuery{"dpkg", "1.7", "source"},
+		&PkgQuery{"dpkg", "1.8", "amd64"}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_source")
 
-	result, err = s.il.Filter([]string{"app", "dpkg (>>1.6.1-3)", "app (>=1.0)", "xyz", "aa (>>3.0)"}, false, nil, 0, nil)
+	result, err = s.il.Filter([]PackageQuery{
+		&DependencyQuery{Dep: Dependency{Pkg: "app"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "xyz"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "aa", Relation: VersionGreater, Version: "3.0"}}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source")
 
-	result, err = s.il.Filter([]string{"app {i386}"}, true, NewPackageList(), 0, []string{"i386"})
+	result, err = s.il.Filter([]PackageQuery{&DependencyQuery{Dep: Dependency{Pkg: "app", Architecture: "i386"}}}, true, NewPackageList(), 0, []string{"i386"})
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386")
 
-	result, err = s.il.Filter([]string{"app (>=0.9)", "lib", "data"}, true, NewPackageList(), 0, []string{"i386", "amd64"})
+	result, err = s.il.Filter([]PackageQuery{
+		&DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "0.9"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "lib"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "data"}}}, true, NewPackageList(), 0, []string{"i386", "amd64"})
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "")
+
+	result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_i386 data_1.1~bp1_all")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{&FieldQuery{Field: "Version", Relation: VersionGreaterOrEqual, Value: "1.0"},
+		&FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 data_1.1~bp1_all")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&NotQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.6.1-3_arm dpkg_1.6.1-3_source dpkg_1.7_source libx_1.5_arm")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionRegexp, Value: "i.*6", Regexp: regexp.MustCompile("i.*6")}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
 }
 
 func (s *PackageListSuite) TestVerifyDependencies(c *C) {

deb/local.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -17,6 +17,10 @@ type LocalRepo struct {
 	Name string
 	// Comment
 	Comment string
+	// DefaultDistribution
+	DefaultDistribution string `codec:",omitempty"`
+	// DefaultComponent
+	DefaultComponent string `codec:",omitempty"`
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 }

deb/local_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"errors"

deb/package.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"fmt"
@@ -24,8 +24,12 @@ type Package struct {
 	Provides []string
 	// Is this source package
 	IsSource bool
+	// Is this udeb package
+	IsUdeb bool
 	// Hash of files section
 	FilesHash uint64
+	// Is this >= 0.6 package?
+	V06Plus bool
 	// Offload fields
 	deps  *PackageDependencies
 	extra *Stanza
@@ -41,6 +45,7 @@ func NewPackageFromControlFile(input Stanza) *Package {
 		Version:      input["Version"],
 		Architecture: input["Architecture"],
 		Source:       input["Source"],
+		V06Plus:      true,
 	}
 
 	delete(input, "Package")
@@ -89,6 +94,7 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 		Version:            input["Version"],
 		Architecture:       "source",
 		SourceArchitecture: input["Architecture"],
+		V06Plus:            true,
 	}
 
 	delete(input, "Package")
@@ -165,9 +171,26 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 	return result, nil
 }
 
+// NewUdebPackageFromControlFile creates .udeb Package from parsed Debian control file
+func NewUdebPackageFromControlFile(input Stanza) *Package {
+	p := NewPackageFromControlFile(input)
+	p.IsUdeb = true
+
+	return p
+}
+
 // Key returns unique key identifying package
 func (p *Package) Key(prefix string) []byte {
-	return []byte(prefix + "P" + p.Architecture + " " + p.Name + " " + p.Version)
+	if p.V06Plus {
+		return []byte(fmt.Sprintf("%sP%s %s %s %08x", prefix, p.Architecture, p.Name, p.Version, p.FilesHash))
+	}
+
+	return p.ShortKey(prefix)
+}
+
+// ShortKey returns key for the package that should be unique in one list
+func (p *Package) ShortKey(prefix string) []byte {
+	return []byte(fmt.Sprintf("%sP%s %s %s", prefix, p.Architecture, p.Name, p.Version))
 }
 
 // String creates readable representation
@@ -175,6 +198,73 @@ func (p *Package) String() string {
 	return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.Architecture)
 }
 
+// GetField returns fields from package
+func (p *Package) GetField(name string) string {
+	switch name {
+	// $Version is handled in FieldQuery
+	case "$Source":
+		if p.IsSource {
+			return ""
+		}
+		source := p.Source
+		if source == "" {
+			return p.Name
+		} else if pos := strings.Index(source, "("); pos != -1 {
+			return strings.TrimSpace(source[:pos])
+		}
+		return source
+	case "$SourceVersion":
+		if p.IsSource {
+			return ""
+		}
+		source := p.Source
+		if pos := strings.Index(source, "("); pos != -1 {
+			if pos2 := strings.LastIndex(source, ")"); pos2 != -1 && pos2 > pos {
+				return strings.TrimSpace(source[pos+1 : pos2])
+			}
+		}
+		return p.Version
+	case "$Architecture":
+		return p.Architecture
+	case "$PackageType":
+		if p.IsSource {
+			return "source"
+		}
+		if p.IsUdeb {
+			return "udeb"
+		}
+		return "deb"
+	case "Name":
+		return p.Name
+	case "Version":
+		return p.Version
+	case "Architecture":
+		if p.IsSource {
+			return p.SourceArchitecture
+		}
+		return p.Architecture
+	case "Source":
+		return p.Source
+	case "Depends":
+		return strings.Join(p.Deps().Depends, ", ")
+	case "Pre-Depends":
+		return strings.Join(p.Deps().PreDepends, ", ")
+	case "Suggests":
+		return strings.Join(p.Deps().Suggests, ", ")
+	case "Recommends":
+		return strings.Join(p.Deps().Recommends, ", ")
+	case "Provides":
+		return strings.Join(p.Provides, ", ")
+	case "Build-Depends":
+		return strings.Join(p.Deps().BuildDepends, ", ")
+	case "Build-Depends-Indep":
+		return strings.Join(p.Deps().BuildDependsInDep, ", ")
+	default:
+		return p.Extra()[name]
+	}
+	return ""
+}
+
 // MatchesArchitecture checks whether packages matches specified architecture
 func (p *Package) MatchesArchitecture(arch string) bool {
 	if p.Architecture == "all" && arch != "source" {
@@ -186,19 +276,23 @@ func (p *Package) MatchesArchitecture(arch string) bool {
 
 // MatchesDependency checks whether package matches specified dependency
 func (p *Package) MatchesDependency(dep Dependency) bool {
-	if dep.Pkg != p.Name {
-		return false
-	}
-
 	if dep.Architecture != "" && !p.MatchesArchitecture(dep.Architecture) {
 		return false
 	}
 
 	if dep.Relation == VersionDontCare {
-		return true
+		if utils.StrSliceHasItem(p.Provides, dep.Pkg) {
+			return true
+		}
+		return dep.Pkg == p.Name
+	}
+
+	if dep.Pkg != p.Name {
+		return false
 	}
 
 	r := CompareVersions(p.Version, dep.Version)
+
 	switch dep.Relation {
 	case VersionEqual:
 		return r == 0
@@ -210,6 +304,11 @@ func (p *Package) MatchesDependency(dep Dependency) bool {
 		return r <= 0
 	case VersionGreaterOrEqual:
 		return r >= 0
+	case VersionPatternMatch:
+		matched, err := filepath.Match(dep.Version, p.Version)
+		return err == nil && matched
+	case VersionRegexp:
+		return dep.Regexp.FindStringIndex(p.Version) != nil
 	}
 
 	panic("unknown relation")
@@ -376,7 +475,8 @@ func (p *Package) Equals(p2 *Package) bool {
 }
 
 // LinkFromPool links package file from pool to dist's pool location
-func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool, prefix string, component string) error {
+func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool,
+	prefix, component string, force bool) error {
 	poolDir, err := p.PoolDirectory()
 	if err != nil {
 		return err
@@ -388,16 +488,18 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 			return err
 		}
 
-		relPath, err := publishedStorage.LinkFromPool(prefix, component, poolDir, packagePool, sourcePath)
+		relPath := filepath.Join("pool", component, poolDir)
+		publishedDirectory := filepath.Join(prefix, relPath)
+
+		err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePath, f.Checksums.MD5, force)
 		if err != nil {
 			return err
 		}
 
-		dir := filepath.Dir(relPath)
 		if p.IsSource {
-			p.Extra()["Directory"] = dir
+			p.Extra()["Directory"] = relPath
 		} else {
-			p.Files()[i].downloadPath = dir
+			p.Files()[i].downloadPath = relPath
 		}
 	}
 
@@ -409,6 +511,8 @@ func (p *Package) PoolDirectory() (string, error) {
 	source := p.Source
 	if source == "" {
 		source = p.Name
+	} else if pos := strings.Index(source, "("); pos != -1 {
+		source = strings.TrimSpace(source[:pos])
 	}
 
 	if len(source) < 2 {

deb/package_collection.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -12,12 +12,19 @@ import (
 type PackageCollection struct {
 	db           database.Storage
 	encodeBuffer bytes.Buffer
+	codecHandle  *codec.MsgpackHandle
 }
 
+// Verify interface
+var (
+	_ PackageCatalog = &PackageCollection{}
+)
+
 // NewPackageCollection creates new PackageCollection and binds it to database
 func NewPackageCollection(db database.Storage) *PackageCollection {
 	return &PackageCollection{
-		db: db,
+		db:          db,
+		codecHandle: &codec.MsgpackHandle{},
 	}
 }
 
@@ -53,7 +60,7 @@ func (collection *PackageCollection) ByKey(key []byte) (*Package, error) {
 	if len(encoded) > 2 && (encoded[0] != 0xc1 || encoded[1] != 0x1) {
 		oldp := &oldPackage{}
 
-		decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+		decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 		err = decoder.Decode(oldp)
 		if err != nil {
 			return nil, err
@@ -83,12 +90,12 @@ func (collection *PackageCollection) ByKey(key []byte) (*Package, error) {
 		p.UpdateFiles(PackageFiles(oldp.Files))
 
 		// Save in new format
-		err = collection.internalUpdate(p)
+		err = collection.Update(p)
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		decoder := codec.NewDecoderBytes(encoded[2:], &codec.MsgpackHandle{})
+		decoder := codec.NewDecoderBytes(encoded[2:], collection.codecHandle)
 		err = decoder.Decode(p)
 		if err != nil {
 			return nil, err
@@ -109,7 +116,7 @@ func (collection *PackageCollection) loadExtra(p *Package) *Stanza {
 
 	stanza := &Stanza{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(stanza)
 	if err != nil {
 		panic("unable to decode extra")
@@ -127,7 +134,7 @@ func (collection *PackageCollection) loadDependencies(p *Package) *PackageDepend
 
 	deps := &PackageDependencies{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(deps)
 	if err != nil {
 		panic("unable to decode deps")
@@ -145,7 +152,7 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 
 	files := &PackageFiles{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(files)
 	if err != nil {
 		panic("unable to decode files")
@@ -156,26 +163,7 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 
 // Update adds or updates information about package in DB checking for conficts first
 func (collection *PackageCollection) Update(p *Package) error {
-	existing, err := collection.ByKey(p.Key(""))
-	if err == nil {
-		// if .Files is different, consider to be conflict
-		if p.FilesHash != existing.FilesHash {
-			return fmt.Errorf("unable to save: %s, conflict with existing packge", p)
-		}
-		// ok, .Files are the same, but maybe some meta-data is different, proceed to saving
-	} else {
-		if err != database.ErrNotFound {
-			return err
-		}
-		// ok, package doesn't exist yet
-	}
-
-	return collection.internalUpdate(p)
-}
-
-// internalUpdate updates information in DB about package and offloaded fields
-func (collection *PackageCollection) internalUpdate(p *Package) error {
-	encoder := codec.NewEncoder(&collection.encodeBuffer, &codec.MsgpackHandle{})
+	encoder := codec.NewEncoder(&collection.encodeBuffer, collection.codecHandle)
 
 	collection.encodeBuffer.Reset()
 	collection.encodeBuffer.WriteByte(0xc1)
@@ -254,3 +242,49 @@ func (collection *PackageCollection) DeleteByKey(key []byte) error {
 	}
 	return nil
 }
+
+// Scan does full scan on all the packages
+func (collection *PackageCollection) Scan(q PackageQuery) (result *PackageList) {
+	result = NewPackageList()
+
+	for _, key := range collection.db.KeysByPrefix([]byte("P")) {
+		pkg, err := collection.ByKey(key)
+		if err != nil {
+			panic(fmt.Sprintf("unable to load package: %s", err))
+		}
+
+		if q.Matches(pkg) {
+			result.Add(pkg)
+		}
+	}
+
+	return
+}
+
+// Search is not implemented
+func (collection *PackageCollection) Search(dep Dependency, allMatches bool) (searchResults []*Package) {
+	panic("Not implemented")
+}
+
+// SearchSupported returns false
+func (collection *PackageCollection) SearchSupported() bool {
+	return false
+}
+
+// SearchByKey finds package by exact key
+func (collection *PackageCollection) SearchByKey(arch, name, version string) (result *PackageList) {
+	result = NewPackageList()
+
+	for _, key := range collection.db.KeysByPrefix([]byte(fmt.Sprintf("P%s %s %s", arch, name, version))) {
+		pkg, err := collection.ByKey(key)
+		if err != nil {
+			panic(fmt.Sprintf("unable to load package: %s", err))
+		}
+
+		if pkg.Architecture == arch && pkg.Name == name && pkg.Version == version {
+			result.Add(pkg)
+		}
+	}
+
+	return
+}

deb/package_collection_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"github.com/smira/aptly/database"
@@ -48,20 +48,6 @@ func (s *PackageCollectionSuite) TestUpdate(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(res.Equals(s.p), Equals, false)
 	c.Assert(res.Equals(p2), Equals, true)
-
-	// change file info
-	p2 = NewPackageFromControlFile(packageStanza.Copy())
-	p2.UpdateFiles(nil)
-	res, err = s.collection.ByKey(p2.Key(""))
-	err = s.collection.Update(p2)
-	c.Assert(err, ErrorMatches, ".*conflict with existing packge")
-	p2 = NewPackageFromControlFile(packageStanza.Copy())
-	files := p2.Files()
-	files[0].Checksums.MD5 = "abcdef"
-	p2.UpdateFiles(files)
-	res, err = s.collection.ByKey(p2.Key(""))
-	err = s.collection.Update(p2)
-	c.Assert(err, ErrorMatches, ".*conflict with existing packge")
 }
 
 func (s *PackageCollectionSuite) TestByKey(c *C) {

deb/package_deps.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"strings"

deb/package_files.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"encoding/binary"

deb/package_files_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"github.com/smira/aptly/files"

deb/package_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -7,6 +7,7 @@ import (
 	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
+	"regexp"
 )
 
 type PackageSuite struct {
@@ -27,6 +28,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
 	c.Check(p.IsSource, Equals, false)
+	c.Check(p.IsUdeb, Equals, false)
 	c.Check(p.Name, Equals, "alien-arena-common")
 	c.Check(p.Version, Equals, "7.40-2")
 	c.Check(p.Architecture, Equals, "i386")
@@ -39,11 +41,27 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 	c.Check(p.deps.Depends, DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)"})
 }
 
+func (s *PackageSuite) TestNewUdebFromPara(c *C) {
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	p := NewUdebPackageFromControlFile(stanza)
+
+	c.Check(p.IsSource, Equals, false)
+	c.Check(p.IsUdeb, Equals, true)
+	c.Check(p.Name, Equals, "dmidecode-udeb")
+	c.Check(p.Version, Equals, "2.11-9")
+	c.Check(p.Architecture, Equals, "amd64")
+	c.Check(p.Provides, DeepEquals, []string(nil))
+	c.Check(p.Files(), HasLen, 1)
+	c.Check(p.Files()[0].Filename, Equals, "dmidecode-udeb_2.11-9_amd64.udeb")
+	c.Check(p.deps.Depends, DeepEquals, []string{"libc6-udeb (>= 2.13)"})
+}
+
 func (s *PackageSuite) TestNewSourceFromPara(c *C) {
 	p, err := NewSourcePackageFromControlFile(s.sourceStanza)
 
 	c.Check(err, IsNil)
 	c.Check(p.IsSource, Equals, true)
+	c.Check(p.IsUdeb, Equals, false)
 	c.Check(p.Name, Equals, "access-modifier-checker")
 	c.Check(p.Version, Equals, "1.0-4")
 	c.Check(p.Architecture, Equals, "source")
@@ -87,10 +105,21 @@ func (s *PackageSuite) TestWithProvides(c *C) {
 func (s *PackageSuite) TestKey(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
+	c.Check(p.Key(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2 c8901eedd79ac51b"))
+	c.Check(p.Key("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2 c8901eedd79ac51b"))
+
+	p.V06Plus = false
 	c.Check(p.Key(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2"))
 	c.Check(p.Key("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2"))
 }
 
+func (s *PackageSuite) TestShortKey(c *C) {
+	p := NewPackageFromControlFile(s.stanza)
+
+	c.Check(p.ShortKey(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2"))
+	c.Check(p.ShortKey("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2"))
+}
+
 func (s *PackageSuite) TestStanza(c *C) {
 	p := NewPackageFromControlFile(s.stanza.Copy())
 	stanza := p.Stanza()
@@ -108,6 +137,65 @@ func (s *PackageSuite) TestString(c *C) {
 	c.Assert(p.String(), Equals, "alien-arena-common_7.40-2_i386")
 }
 
+func (s *PackageSuite) TestGetField(c *C) {
+	p := NewPackageFromControlFile(s.stanza.Copy())
+
+	stanza2 := s.stanza.Copy()
+	delete(stanza2, "Source")
+	stanza2["Provides"] = "app, game"
+	p2 := NewPackageFromControlFile(stanza2)
+
+	stanza3 := s.stanza.Copy()
+	stanza3["Source"] = "alien-arena (3.5)"
+	p3 := NewPackageFromControlFile(stanza3)
+
+	p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
+
+	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	p5 := NewUdebPackageFromControlFile(stanza5)
+
+	c.Check(p.GetField("$Source"), Equals, "alien-arena")
+	c.Check(p2.GetField("$Source"), Equals, "alien-arena-common")
+	c.Check(p3.GetField("$Source"), Equals, "alien-arena")
+	c.Check(p4.GetField("$Source"), Equals, "")
+	c.Check(p5.GetField("$Source"), Equals, "dmidecode")
+
+	c.Check(p.GetField("$SourceVersion"), Equals, "7.40-2")
+	c.Check(p2.GetField("$SourceVersion"), Equals, "7.40-2")
+	c.Check(p3.GetField("$SourceVersion"), Equals, "3.5")
+	c.Check(p4.GetField("$SourceVersion"), Equals, "")
+	c.Check(p5.GetField("$SourceVersion"), Equals, "2.11-9")
+
+	c.Check(p.GetField("$Architecture"), Equals, "i386")
+	c.Check(p4.GetField("$Architecture"), Equals, "source")
+	c.Check(p5.GetField("$Architecture"), Equals, "amd64")
+
+	c.Check(p.GetField("$PackageType"), Equals, "deb")
+	c.Check(p4.GetField("$PackageType"), Equals, "source")
+	c.Check(p5.GetField("$PackageType"), Equals, "udeb")
+
+	c.Check(p.GetField("Name"), Equals, "alien-arena-common")
+	c.Check(p4.GetField("Name"), Equals, "access-modifier-checker")
+
+	c.Check(p.GetField("Architecture"), Equals, "i386")
+	c.Check(p4.GetField("Architecture"), Equals, "all")
+
+	c.Check(p.GetField("Version"), Equals, "7.40-2")
+
+	c.Check(p.GetField("Source"), Equals, "alien-arena")
+	c.Check(p2.GetField("Source"), Equals, "")
+	c.Check(p3.GetField("Source"), Equals, "alien-arena (3.5)")
+	c.Check(p4.GetField("Source"), Equals, "")
+
+	c.Check(p.GetField("Depends"), Equals, "libc6 (>= 2.7), alien-arena-data (>= 7.40)")
+
+	c.Check(p.GetField("Provides"), Equals, "")
+	c.Check(p2.GetField("Provides"), Equals, "app, game")
+
+	c.Check(p.GetField("Section"), Equals, "contrib/games")
+	c.Check(p.GetField("Priority"), Equals, "extra")
+}
+
 func (s *PackageSuite) TestEquals(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
@@ -163,6 +251,9 @@ func (s *PackageSuite) TestMatchesDependency(c *C) {
 	// exact match
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionEqual, Version: "7.40-2"}), Equals, true)
 
+	// exact match, same version, no revision specified
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionEqual, Version: "7.40"}), Equals, false)
+
 	// different name
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena", Architecture: "i386", Relation: VersionEqual, Version: "7.40-2"}), Equals, false)
 
@@ -193,6 +284,29 @@ func (s *PackageSuite) TestMatchesDependency(c *C) {
 	// <=
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionLessOrEqual, Version: "7.40-2"}), Equals, true)
 	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionLessOrEqual, Version: "7.40-1"}), Equals, false)
+
+	// %
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-*"}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-[2]"}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-[2"}), Equals, false)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionPatternMatch, Version: "7.40-[34]"}), Equals, false)
+
+	// ~
+	c.Check(
+		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
+			Regexp: regexp.MustCompile("7\\.40-.*")}), Equals, true)
+	c.Check(
+		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
+			Regexp: regexp.MustCompile("40")}), Equals, true)
+	c.Check(
+		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
+			Regexp: regexp.MustCompile("39-.*")}), Equals, false)
+
+	// Provides
+	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Relation: VersionDontCare}), Equals, false)
+	p.Provides = []string{"fun", "game"}
+	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Relation: VersionDontCare}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Architecture: "amd64", Relation: VersionDontCare}), Equals, false)
 }
 
 func (s *PackageSuite) TestGetDependencies(c *C) {
@@ -230,6 +344,12 @@ func (s *PackageSuite) TestPoolDirectory(c *C) {
 	c.Check(err, IsNil)
 	c.Check(dir, Equals, "liba/libarena")
 
+	p = NewPackageFromControlFile(packageStanza.Copy())
+	p.Source = "gcc-defaults (1.77)"
+	dir, err = p.PoolDirectory()
+	c.Check(err, IsNil)
+	c.Check(dir, Equals, "g/gcc-defaults")
+
 	p = NewPackageFromControlFile(packageStanza.Copy())
 	p.Source = "l"
 	_, err = p.PoolDirectory()
@@ -249,13 +369,13 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
 	c.Assert(err, IsNil)
 	file.Close()
 
-	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free")
+	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
 	c.Check(err, IsNil)
 	c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
 	c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena")
 
 	p.IsSource = true
-	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free")
+	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
 	c.Check(err, IsNil)
 	c.Check(p.Extra()["Directory"], Equals, "pool/non-free/a/alien-arena")
 }
@@ -359,3 +479,20 @@ Directory: pool/main/a/access-modifier-checker
 Priority: source
 Section: java
 `
+
+const udebPackageMeta = `Package: dmidecode-udeb
+Source: dmidecode
+Version: 2.11-9
+Installed-Size: 115
+Maintainer: Daniel Baumann <daniel.baumann@progress-technologies.net>
+Architecture: amd64
+Depends: libc6-udeb (>= 2.13)
+Description: SMBIOS/DMI table decoder (udeb)
+Description-md5: bdfb786c6a57097be8c8600b800e749f
+Section: debian-installer
+Priority: optional
+Filename: pool/main/d/dmidecode/dmidecode-udeb_2.11-9_amd64.udeb
+Size: 29188
+MD5sum: ae70341c4d96dcded89fa670bcfea31e
+SHA1: 9532ae4226a85805189a671ee0283f719d48a5ba
+SHA256: bbb3a2cb07f741c3995b6d4bb08d772d83582b93a0236d4ea7736bc0370fc320`

deb/ppa.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"fmt"
@@ -11,14 +11,14 @@ import (
 var ppaRegexp = regexp.MustCompile("^ppa:([^/]+)/(.+)$")
 
 // ParsePPA converts ppa URL like ppa:user/ppa-name to full HTTP url
-func ParsePPA(ppaURL string) (url string, distribution string, components []string, err error) {
+func ParsePPA(ppaURL string, config *utils.ConfigStructure) (url string, distribution string, components []string, err error) {
 	matches := ppaRegexp.FindStringSubmatch(ppaURL)
 	if matches == nil {
 		err = fmt.Errorf("unable to parse ppa URL: %v", ppaURL)
 		return
 	}
 
-	distributorID := utils.Config.PpaDistributorID
+	distributorID := config.PpaDistributorID
 	if distributorID == "" {
 		distributorID, err = getDistributorID()
 		if err != nil {
@@ -27,7 +27,7 @@ func ParsePPA(ppaURL string) (url string, distribution string, components []stri
 		}
 	}
 
-	codename := utils.Config.PpaCodename
+	codename := config.PpaCodename
 	if codename == "" {
 		codename, err = getCodename()
 		if err != nil {

deb/ppa_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"github.com/smira/aptly/utils"
@@ -6,27 +6,19 @@ import (
 )
 
 type PpaSuite struct {
-	savedConfig utils.ConfigStructure
+	config utils.ConfigStructure
 }
 
 var _ = Suite(&PpaSuite{})
 
-func (s *PpaSuite) SetUpTest(c *C) {
-	s.savedConfig = utils.Config
-}
-
-func (s *PpaSuite) TearDownTest(c *C) {
-	utils.Config = s.savedConfig
-}
-
 func (s *PpaSuite) TestParsePPA(c *C) {
-	_, _, _, err := ParsePPA("ppa:dedeed")
+	_, _, _, err := ParsePPA("ppa:dedeed", &s.config)
 	c.Check(err, ErrorMatches, "unable to parse ppa URL.*")
 
-	utils.Config.PpaDistributorID = "debian"
-	utils.Config.PpaCodename = "wheezy"
+	s.config.PpaDistributorID = "debian"
+	s.config.PpaCodename = "wheezy"
 
-	url, distribution, components, err := ParsePPA("ppa:user/project")
+	url, distribution, components, err := ParsePPA("ppa:user/project", &s.config)
 	c.Check(err, IsNil)
 	c.Check(url, Equals, "http://ppa.launchpad.net/user/project/debian")
 	c.Check(distribution, Equals, "wheezy")

deb/publish_test.go

@@ -0,0 +1,818 @@
+package deb
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/files"
+	"github.com/ugorji/go/codec"
+	"io/ioutil"
+	. "launchpad.net/gocheck"
+	"os"
+	"path/filepath"
+)
+
+type pathExistsChecker struct {
+	*CheckerInfo
+}
+
+var PathExists = &pathExistsChecker{
+	&CheckerInfo{Name: "PathExists", Params: []string{"path"}},
+}
+
+func (checker *pathExistsChecker) Check(params []interface{}, names []string) (result bool, error string) {
+	_, err := os.Stat(params[0].(string))
+	return err == nil, ""
+}
+
+type NullSigner struct{}
+
+func (n *NullSigner) Init() error {
+	return nil
+}
+
+func (n *NullSigner) SetKey(keyRef string) {
+}
+
+func (n *NullSigner) SetKeyRing(keyring, secretKeyring string) {
+}
+
+func (n *NullSigner) SetPassphrase(passphrase, passphraseFile string) {
+}
+
+func (n *NullSigner) DetachedSign(source string, destination string) error {
+	return ioutil.WriteFile(destination, []byte{}, 0644)
+}
+
+func (n *NullSigner) ClearSign(source string, destination string) error {
+	return ioutil.WriteFile(destination, []byte{}, 0644)
+}
+
+type FakeStorageProvider struct {
+	storages map[string]aptly.PublishedStorage
+}
+
+func (p *FakeStorageProvider) GetPublishedStorage(name string) aptly.PublishedStorage {
+	storage, ok := p.storages[name]
+	if !ok {
+		panic(fmt.Sprintf("unknown storage: %#v", name))
+	}
+	return storage
+}
+
+type PublishedRepoSuite struct {
+	PackageListMixinSuite
+	repo, repo2, repo3, repo4, repo5    *PublishedRepo
+	root, root2                         string
+	provider                            *FakeStorageProvider
+	publishedStorage, publishedStorage2 *files.PublishedStorage
+	packagePool                         aptly.PackagePool
+	localRepo                           *LocalRepo
+	snapshot, snapshot2                 *Snapshot
+	db                                  database.Storage
+	factory                             *CollectionFactory
+	packageCollection                   *PackageCollection
+}
+
+var _ = Suite(&PublishedRepoSuite{})
+
+func (s *PublishedRepoSuite) SetUpTest(c *C) {
+	s.SetUpPackages()
+
+	s.db, _ = database.OpenDB(c.MkDir())
+	s.factory = NewCollectionFactory(s.db)
+
+	s.root = c.MkDir()
+	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.root2 = c.MkDir()
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{
+		"":            s.publishedStorage,
+		"files:other": s.publishedStorage2}}
+	s.packagePool = files.NewPackagePool(s.root)
+
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo.packageRefs = s.reflist
+	s.factory.RemoteRepoCollection().Add(repo)
+
+	s.localRepo = NewLocalRepo("local1", "comment1")
+	s.localRepo.packageRefs = s.reflist
+	s.factory.LocalRepoCollection().Add(s.localRepo)
+
+	s.snapshot, _ = NewSnapshotFromRepository("snap", repo)
+	s.factory.SnapshotCollection().Add(s.snapshot)
+
+	s.snapshot2, _ = NewSnapshotFromRepository("snap", repo)
+	s.factory.SnapshotCollection().Add(s.snapshot2)
+
+	s.packageCollection = s.factory.PackageCollection()
+	s.packageCollection.Update(s.p1)
+	s.packageCollection.Update(s.p2)
+	s.packageCollection.Update(s.p3)
+
+	s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+
+	s.repo2, _ = NewPublishedRepo("", "ppa", "maverick", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+
+	s.repo3, _ = NewPublishedRepo("", "linux", "natty", nil, []string{"main", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+
+	s.repo4, _ = NewPublishedRepo("", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+
+	poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
+	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
+	f, err := os.Create(poolPath)
+	c.Assert(err, IsNil)
+	f.Close()
+}
+
+func (s *PublishedRepoSuite) TearDownTest(c *C) {
+	s.db.Close()
+}
+
+func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) {
+	c.Check(s.repo.sourceItems["main"].snapshot, Equals, s.snapshot)
+	c.Check(s.repo.SourceKind, Equals, "snapshot")
+	c.Check(s.repo.Sources["main"], Equals, s.snapshot.UUID)
+	c.Check(s.repo.Components(), DeepEquals, []string{"main"})
+
+	c.Check(s.repo2.sourceItems["main"].localRepo, Equals, s.localRepo)
+	c.Check(s.repo2.SourceKind, Equals, "local")
+	c.Check(s.repo2.Sources["main"], Equals, s.localRepo.UUID)
+	c.Check(s.repo2.sourceItems["main"].packageRefs.Len(), Equals, 3)
+	c.Check(s.repo2.Components(), DeepEquals, []string{"main"})
+
+	c.Check(s.repo.RefList("main").Len(), Equals, 3)
+	c.Check(s.repo2.RefList("main").Len(), Equals, 3)
+
+	c.Check(s.repo3.Sources, DeepEquals, map[string]string{"main": s.snapshot.UUID, "contrib": s.snapshot2.UUID})
+	c.Check(s.repo3.SourceKind, Equals, "snapshot")
+	c.Check(s.repo3.sourceItems["main"].snapshot, Equals, s.snapshot)
+	c.Check(s.repo3.sourceItems["contrib"].snapshot, Equals, s.snapshot2)
+	c.Check(s.repo3.Components(), DeepEquals, []string{"contrib", "main"})
+
+	c.Check(s.repo3.RefList("main").Len(), Equals, 3)
+	c.Check(s.repo3.RefList("contrib").Len(), Equals, 3)
+
+	c.Check(func() { NewPublishedRepo("", ".", "a", nil, nil, nil, s.factory) }, PanicMatches, "publish with empty sources")
+	c.Check(func() {
+		NewPublishedRepo("", ".", "a", nil, []string{"main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	}, PanicMatches, "sources and components should be equal in size")
+	c.Check(func() {
+		NewPublishedRepo("", ".", "a", nil, []string{"main", "contrib"}, []interface{}{s.localRepo, s.snapshot2}, s.factory)
+	}, PanicMatches, "interface conversion:.*")
+
+	_, err := NewPublishedRepo("", ".", "a", nil, []string{"main", "main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	c.Check(err, ErrorMatches, "duplicate component name: main")
+
+	_, err = NewPublishedRepo("", ".", "wheezy/updates", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	c.Check(err, ErrorMatches, "invalid distribution wheezy/updates, '/' is not allowed")
+}
+
+func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
+
+	for _, t := range []struct {
+		prefix        string
+		expected      string
+		errorExpected string
+	}{
+		{
+			prefix:   "ppa",
+			expected: "ppa",
+		},
+		{
+			prefix:   "",
+			expected: ".",
+		},
+		{
+			prefix:   "/",
+			expected: ".",
+		},
+		{
+			prefix:   "//",
+			expected: ".",
+		},
+		{
+			prefix:   "//ppa/",
+			expected: "ppa",
+		},
+		{
+			prefix:   "ppa/..",
+			expected: ".",
+		},
+		{
+			prefix:   "ppa/ubuntu/",
+			expected: "ppa/ubuntu",
+		},
+		{
+			prefix:   "ppa/../ubuntu/",
+			expected: "ubuntu",
+		},
+		{
+			prefix:        "../ppa/",
+			errorExpected: "invalid prefix .*",
+		},
+		{
+			prefix:        "../ppa/../ppa/",
+			errorExpected: "invalid prefix .*",
+		},
+		{
+			prefix:        "ppa/dists",
+			errorExpected: "invalid prefix .*",
+		},
+		{
+			prefix:        "ppa/pool",
+			errorExpected: "invalid prefix .*",
+		},
+	} {
+		repo, err := NewPublishedRepo("", t.prefix, "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+		if t.errorExpected != "" {
+			c.Check(err, ErrorMatches, t.errorExpected)
+		} else {
+			c.Check(repo.Prefix, Equals, t.expected)
+		}
+	}
+}
+
+func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
+	repo, err := NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "squeeze")
+	c.Check(repo.Components(), DeepEquals, []string{"main"})
+
+	repo, err = NewPublishedRepo("", "ppa", "wheezy", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "wheezy")
+	c.Check(repo.Components(), DeepEquals, []string{"main"})
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"non-free"}, []interface{}{s.snapshot}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "squeeze")
+	c.Check(repo.Components(), DeepEquals, []string{"non-free"})
+
+	repo, err = NewPublishedRepo("", "ppa", "squeeze", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "squeeze")
+	c.Check(repo.Components(), DeepEquals, []string{"main"})
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	c.Check(err, ErrorMatches, "unable to guess distribution name, please specify explicitly")
+
+	s.localRepo.DefaultDistribution = "precise"
+	s.localRepo.DefaultComponent = "contrib"
+	s.factory.LocalRepoCollection().Update(s.localRepo)
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "precise")
+	c.Check(repo.Components(), DeepEquals, []string{"contrib"})
+
+	s.localRepo.DefaultDistribution = "precise/updates"
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "precise-updates")
+	c.Check(repo.Components(), DeepEquals, []string{"contrib"})
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "squeeze")
+	c.Check(repo.Components(), DeepEquals, []string{"contrib", "main"})
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	c.Check(err, ErrorMatches, "duplicate component name: main")
+}
+
+func (s *PublishedRepoSuite) TestPublish(c *C) {
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false)
+	c.Assert(err, IsNil)
+
+	c.Check(s.repo.Architectures, DeepEquals, []string{"i386"})
+
+	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
+	c.Assert(err, IsNil)
+
+	cfr := NewControlFileReader(rf)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(st["Origin"], Equals, "ppa squeeze")
+	c.Check(st["Components"], Equals, "main")
+	c.Check(st["Architectures"], Equals, "i386")
+
+	pf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages"))
+	c.Assert(err, IsNil)
+
+	cfr = NewControlFileReader(pf)
+
+	for i := 0; i < 3; i++ {
+		st, err = cfr.ReadStanza()
+		c.Assert(err, IsNil)
+
+		c.Check(st["Filename"], Equals, "pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
+	}
+
+	st, err = cfr.ReadStanza()
+	c.Assert(err, IsNil)
+	c.Assert(st, IsNil)
+
+	drf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"))
+	c.Assert(err, IsNil)
+
+	cfr = NewControlFileReader(drf)
+	st, err = cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(st["Archive"], Equals, "squeeze")
+	c.Check(st["Architecture"], Equals, "i386")
+
+	_, err = os.Stat(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb"))
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	c.Assert(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"), PathExists)
+}
+
+func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) {
+	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	c.Assert(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/main/binary-i386/Release"), PathExists)
+}
+
+func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) {
+	err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	c.Assert(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/main/source/Release"), PathExists)
+}
+
+func (s *PublishedRepoSuite) TestPublishOtherStorage(c *C) {
+	err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil, false)
+	c.Assert(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/maverick/Release"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), Not(PathExists))
+}
+
+func (s *PublishedRepoSuite) TestString(c *C) {
+	c.Check(s.repo.String(), Equals,
+		"ppa/squeeze [] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	c.Check(s.repo2.String(), Equals,
+		"ppa/maverick [] publishes {main: [local1]: comment1}")
+	repo, _ := NewPublishedRepo("", "", "squeeze", []string{"s390"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	c.Check(repo.String(), Equals,
+		"./squeeze [s390] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	repo, _ = NewPublishedRepo("", "", "squeeze", []string{"i386", "amd64"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	c.Check(repo.String(), Equals,
+		"./squeeze [i386, amd64] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	repo.Origin = "myorigin"
+	c.Check(repo.String(), Equals,
+		"./squeeze (origin: myorigin) [i386, amd64] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	repo.Label = "mylabel"
+	c.Check(repo.String(), Equals,
+		"./squeeze (origin: myorigin, label: mylabel) [i386, amd64] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	c.Check(s.repo3.String(), Equals,
+		"linux/natty [] publishes {contrib: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}, {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
+	c.Check(s.repo5.String(), Equals,
+		"files:other:ppa/maverick [source] publishes {main: [local1]: comment1}")
+}
+
+func (s *PublishedRepoSuite) TestKey(c *C) {
+	c.Check(s.repo.Key(), DeepEquals, []byte("Uppa>>squeeze"))
+	c.Check(s.repo5.Key(), DeepEquals, []byte("Ufiles:other:ppa>>maverick"))
+}
+
+func (s *PublishedRepoSuite) TestRefKey(c *C) {
+	c.Check(s.repo.RefKey(""), DeepEquals, []byte("E"+s.repo.UUID))
+	c.Check(s.repo.RefKey("main"), DeepEquals, []byte("E"+s.repo.UUID+"main"))
+}
+
+func (s *PublishedRepoSuite) TestEncodeDecode(c *C) {
+	encoded := s.repo.Encode()
+	repo := &PublishedRepo{}
+	err := repo.Decode(encoded)
+
+	s.repo.sourceItems = nil
+	c.Assert(err, IsNil)
+	c.Assert(repo, DeepEquals, s.repo)
+
+	encoded2 := s.repo2.Encode()
+	repo2 := &PublishedRepo{}
+	err = repo2.Decode(encoded2)
+
+	s.repo2.sourceItems = nil
+	c.Assert(err, IsNil)
+	c.Assert(repo2, DeepEquals, s.repo2)
+}
+
+type PublishedRepoCollectionSuite struct {
+	PackageListMixinSuite
+	db                                database.Storage
+	factory                           *CollectionFactory
+	snapshotCollection                *SnapshotCollection
+	collection                        *PublishedRepoCollection
+	snap1, snap2                      *Snapshot
+	localRepo                         *LocalRepo
+	repo1, repo2, repo3, repo4, repo5 *PublishedRepo
+}
+
+var _ = Suite(&PublishedRepoCollectionSuite{})
+
+func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
+	s.db, _ = database.OpenDB(c.MkDir())
+	s.factory = NewCollectionFactory(s.db)
+
+	s.snapshotCollection = s.factory.SnapshotCollection()
+
+	s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
+	s.snap2 = NewSnapshotFromPackageList("snap2", []*Snapshot{}, NewPackageList(), "desc2")
+
+	s.snapshotCollection.Add(s.snap1)
+	s.snapshotCollection.Add(s.snap2)
+
+	s.localRepo = NewLocalRepo("local1", "comment1")
+	s.factory.LocalRepoCollection().Add(s.localRepo)
+
+	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main", "contrib"}, []interface{}{s.snap2, s.snap1}, s.factory)
+	s.repo3, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap2}, s.factory)
+	s.repo4, _ = NewPublishedRepo("", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+
+	s.collection = s.factory.PublishedRepoCollection()
+}
+
+func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) {
+	s.db.Close()
+}
+
+func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C) {
+	r, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Assert(err, ErrorMatches, "*.not found")
+
+	c.Assert(s.collection.Add(s.repo1), IsNil)
+	c.Assert(s.collection.Add(s.repo1), ErrorMatches, ".*already exists")
+	c.Assert(s.collection.CheckDuplicate(s.repo2), IsNil)
+	c.Assert(s.collection.Add(s.repo2), IsNil)
+	c.Assert(s.collection.Add(s.repo3), ErrorMatches, ".*already exists")
+	c.Assert(s.collection.CheckDuplicate(s.repo3), Equals, s.repo1)
+	c.Assert(s.collection.Add(s.repo4), IsNil)
+	c.Assert(s.collection.Add(s.repo5), IsNil)
+
+	r, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Assert(err, IsNil)
+
+	err = s.collection.LoadComplete(r, s.factory)
+	c.Assert(err, IsNil)
+	c.Assert(r.String(), Equals, s.repo1.String())
+
+	collection := NewPublishedRepoCollection(s.db)
+	r, err = collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Assert(err, IsNil)
+
+	err = s.collection.LoadComplete(r, s.factory)
+	c.Assert(err, IsNil)
+	c.Assert(r.String(), Equals, s.repo1.String())
+
+	r, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "precise")
+	c.Check(r.String(), Equals, s.repo5.String())
+}
+
+func (s *PublishedRepoCollectionSuite) TestByUUID(c *C) {
+	r, err := s.collection.ByUUID(s.repo1.UUID)
+	c.Assert(err, ErrorMatches, "*.not found")
+
+	c.Assert(s.collection.Add(s.repo1), IsNil)
+
+	r, err = s.collection.ByUUID(s.repo1.UUID)
+	c.Assert(err, IsNil)
+
+	err = s.collection.LoadComplete(r, s.factory)
+	c.Assert(err, IsNil)
+	c.Assert(r.String(), Equals, s.repo1.String())
+}
+
+func (s *PublishedRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
+	c.Assert(s.collection.Update(s.repo1), IsNil)
+	c.Assert(s.collection.Update(s.repo4), IsNil)
+
+	collection := NewPublishedRepoCollection(s.db)
+	r, err := collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Assert(err, IsNil)
+	c.Assert(r.sourceItems["main"].snapshot, IsNil)
+	c.Assert(s.collection.LoadComplete(r, s.factory), IsNil)
+	c.Assert(r.Sources["main"], Equals, s.repo1.sourceItems["main"].snapshot.UUID)
+	c.Assert(r.RefList("main").Len(), Equals, 0)
+
+	r, err = collection.ByStoragePrefixDistribution("", "ppa", "precise")
+	c.Assert(err, IsNil)
+	c.Assert(r.sourceItems["main"].localRepo, IsNil)
+	c.Assert(s.collection.LoadComplete(r, s.factory), IsNil)
+	c.Assert(r.sourceItems["main"].localRepo.UUID, Equals, s.repo4.sourceItems["main"].localRepo.UUID)
+	c.Assert(r.sourceItems["main"].packageRefs.Len(), Equals, 0)
+	c.Assert(r.RefList("main").Len(), Equals, 0)
+}
+
+func (s *PublishedRepoCollectionSuite) TestLoadPre0_6(c *C) {
+	type oldPublishedRepo struct {
+		UUID          string
+		Prefix        string
+		Distribution  string
+		Origin        string
+		Label         string
+		Architectures []string
+		SourceKind    string
+		Component     string
+		SourceUUID    string `codec:"SnapshotUUID"`
+	}
+
+	old := oldPublishedRepo{
+		UUID:          s.repo1.UUID,
+		Prefix:        "ppa",
+		Distribution:  "anaconda",
+		Architectures: []string{"i386"},
+		SourceKind:    "local",
+		Component:     "contrib",
+		SourceUUID:    s.localRepo.UUID,
+	}
+
+	var buf bytes.Buffer
+
+	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
+	encoder.Encode(&old)
+
+	c.Assert(s.db.Put(s.repo1.Key(), buf.Bytes()), IsNil)
+	c.Assert(s.db.Put(s.repo1.RefKey(""), s.localRepo.RefList().Encode()), IsNil)
+
+	collection := NewPublishedRepoCollection(s.db)
+	repo, err := collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, IsNil)
+	c.Check(repo.Component, Equals, "")
+	c.Check(repo.SourceUUID, Equals, "")
+	c.Check(repo.Sources, DeepEquals, map[string]string{"contrib": s.localRepo.UUID})
+
+	c.Check(collection.LoadComplete(repo, s.factory), IsNil)
+	c.Check(repo.sourceItems["contrib"].localRepo.UUID, Equals, s.localRepo.UUID)
+	c.Check(repo.RefList("contrib").Len(), Equals, 0)
+}
+
+func (s *PublishedRepoCollectionSuite) TestForEachAndLen(c *C) {
+	s.collection.Add(s.repo1)
+
+	count := 0
+	err := s.collection.ForEach(func(*PublishedRepo) error {
+		count++
+		return nil
+	})
+	c.Assert(count, Equals, 1)
+	c.Assert(err, IsNil)
+
+	c.Check(s.collection.Len(), Equals, 1)
+
+	e := errors.New("c")
+
+	err = s.collection.ForEach(func(*PublishedRepo) error {
+		return e
+	})
+	c.Assert(err, Equals, e)
+}
+
+func (s *PublishedRepoCollectionSuite) TestBySnapshot(c *C) {
+	c.Check(s.collection.Add(s.repo1), IsNil)
+	c.Check(s.collection.Add(s.repo2), IsNil)
+
+	c.Check(s.collection.BySnapshot(s.snap1), DeepEquals, []*PublishedRepo{s.repo1, s.repo2})
+	c.Check(s.collection.BySnapshot(s.snap2), DeepEquals, []*PublishedRepo{s.repo2})
+}
+
+func (s *PublishedRepoCollectionSuite) TestByLocalRepo(c *C) {
+	c.Check(s.collection.Add(s.repo1), IsNil)
+	c.Check(s.collection.Add(s.repo4), IsNil)
+	c.Check(s.collection.Add(s.repo5), IsNil)
+
+	c.Check(s.collection.ByLocalRepo(s.localRepo), DeepEquals, []*PublishedRepo{s.repo4, s.repo5})
+}
+
+type PublishedRepoRemoveSuite struct {
+	PackageListMixinSuite
+	db                                  database.Storage
+	factory                             *CollectionFactory
+	snapshotCollection                  *SnapshotCollection
+	collection                          *PublishedRepoCollection
+	root, root2                         string
+	provider                            *FakeStorageProvider
+	publishedStorage, publishedStorage2 *files.PublishedStorage
+	snap1                               *Snapshot
+	repo1, repo2, repo3, repo4, repo5   *PublishedRepo
+}
+
+var _ = Suite(&PublishedRepoRemoveSuite{})
+
+func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
+	s.db, _ = database.OpenDB(c.MkDir())
+	s.factory = NewCollectionFactory(s.db)
+
+	s.snapshotCollection = s.factory.SnapshotCollection()
+
+	s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
+
+	s.snapshotCollection.Add(s.snap1)
+
+	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo3, _ = NewPublishedRepo("", "ppa", "meduza", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo4, _ = NewPublishedRepo("", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory)
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory)
+
+	s.collection = s.factory.PublishedRepoCollection()
+	s.collection.Add(s.repo1)
+	s.collection.Add(s.repo2)
+	s.collection.Add(s.repo3)
+	s.collection.Add(s.repo4)
+	s.collection.Add(s.repo5)
+
+	s.root = c.MkDir()
+	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.publishedStorage.MkDir("ppa/dists/anaconda")
+	s.publishedStorage.MkDir("ppa/dists/meduza")
+	s.publishedStorage.MkDir("ppa/dists/osminog")
+	s.publishedStorage.MkDir("ppa/pool/main")
+	s.publishedStorage.MkDir("ppa/pool/contrib")
+	s.publishedStorage.MkDir("dists/anaconda")
+	s.publishedStorage.MkDir("pool/main")
+
+	s.root2 = c.MkDir()
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.publishedStorage2.MkDir("ppa/dists/osminog")
+	s.publishedStorage2.MkDir("ppa/pool/contrib")
+
+	s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{
+		"":            s.publishedStorage,
+		"files:other": s.publishedStorage2}}
+}
+
+func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
+	s.db.Close()
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
+	s.repo1.RemoveFiles(s.provider, false, []string{}, nil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
+	s.repo1.RemoveFiles(s.provider, false, []string{"main"}, nil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithTwoPools(c *C) {
+	s.repo1.RemoveFiles(s.provider, false, []string{"main", "contrib"}, nil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
+	s.repo1.RemoveFiles(s.provider, true, []string{"main"}, nil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
+	s.repo2.RemoveFiles(s.provider, true, []string{"main"}, nil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	collection := NewPublishedRepoCollection(s.db)
+	_, err = collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
+	c.Check(err, ErrorMatches, ".*not found")
+
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil)
+	c.Check(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
+	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	collection := NewPublishedRepoCollection(s.db)
+	_, err = collection.ByStoragePrefixDistribution("", ".", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) {
+	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	collection := NewPublishedRepoCollection(s.db)
+	_, err = collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
+}

deb/query.go

@@ -0,0 +1,267 @@
+package deb
+
+import (
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// PackageCatalog is abstraction on top of PackageCollection and PackageList
+type PackageCatalog interface {
+	Scan(q PackageQuery) (result *PackageList)
+	Search(dep Dependency, allMatches bool) (searchResults []*Package)
+	SearchSupported() bool
+	SearchByKey(arch, name, version string) (result *PackageList)
+}
+
+// PackageQuery is interface of predicate on Package
+type PackageQuery interface {
+	// Matches calculates match of condition against package
+	Matches(pkg *Package) bool
+	// Fast returns if search strategy is possible for this query
+	Fast(list PackageCatalog) bool
+	// Query performs search on package list
+	Query(list PackageCatalog) *PackageList
+	// String interface
+	String() string
+}
+
+// OrQuery is L | R
+type OrQuery struct {
+	L, R PackageQuery
+}
+
+// AndQuery is L , R
+type AndQuery struct {
+	L, R PackageQuery
+}
+
+// NotQuery is ! Q
+type NotQuery struct {
+	Q PackageQuery
+}
+
+// FieldQuery is generic request against field
+type FieldQuery struct {
+	Field    string
+	Relation int
+	Value    string
+	Regexp   *regexp.Regexp `codec:"-"`
+}
+
+// PkgQuery is search request against specific package
+type PkgQuery struct {
+	Pkg     string
+	Version string
+	Arch    string
+}
+
+// DependencyQuery is generic Debian-dependency like query
+type DependencyQuery struct {
+	Dep Dependency
+}
+
+// Matches if any of L, R matches
+func (q *OrQuery) Matches(pkg *Package) bool {
+	return q.L.Matches(pkg) || q.R.Matches(pkg)
+}
+
+// Fast is true only if both parts are fast
+func (q *OrQuery) Fast(list PackageCatalog) bool {
+	return q.L.Fast(list) && q.R.Fast(list)
+}
+
+// Query strategy depends on nodes
+func (q *OrQuery) Query(list PackageCatalog) (result *PackageList) {
+	if q.Fast(list) {
+		result = q.L.Query(list)
+		result.Append(q.R.Query(list))
+	} else {
+		result = list.Scan(q)
+	}
+	return
+}
+
+// String interface
+func (q *OrQuery) String() string {
+	return fmt.Sprintf("(%s) | (%s)", q.L, q.R)
+}
+
+// Matches if both of L, R matches
+func (q *AndQuery) Matches(pkg *Package) bool {
+	return q.L.Matches(pkg) && q.R.Matches(pkg)
+}
+
+// Fast is true if any of the parts are fast
+func (q *AndQuery) Fast(list PackageCatalog) bool {
+	return q.L.Fast(list) || q.R.Fast(list)
+}
+
+// Query strategy depends on nodes
+func (q *AndQuery) Query(list PackageCatalog) (result *PackageList) {
+	if !q.Fast(list) {
+		result = list.Scan(q)
+	} else {
+		if q.L.Fast(list) {
+			result = q.L.Query(list)
+			result = result.Scan(q.R)
+		} else {
+			result = q.R.Query(list)
+			result = result.Scan(q.L)
+		}
+	}
+	return
+}
+
+// String interface
+func (q *AndQuery) String() string {
+	return fmt.Sprintf("(%s), (%s)", q.L, q.R)
+}
+
+// Matches if not matches
+func (q *NotQuery) Matches(pkg *Package) bool {
+	return !q.Q.Matches(pkg)
+}
+
+// Fast is false
+func (q *NotQuery) Fast(list PackageCatalog) bool {
+	return false
+}
+
+// Query strategy is scan always
+func (q *NotQuery) Query(list PackageCatalog) (result *PackageList) {
+	result = list.Scan(q)
+	return
+}
+
+// String interface
+func (q *NotQuery) String() string {
+	return fmt.Sprintf("!(%s)", q.Q)
+}
+
+// Matches on generic field
+func (q *FieldQuery) Matches(pkg *Package) bool {
+	if q.Field == "$Version" {
+		return pkg.MatchesDependency(Dependency{Pkg: pkg.Name, Relation: q.Relation, Version: q.Value, Regexp: q.Regexp})
+	}
+	if q.Field == "$Architecture" && q.Relation == VersionEqual {
+		return pkg.MatchesArchitecture(q.Value)
+	}
+
+	field := pkg.GetField(q.Field)
+
+	switch q.Relation {
+	case VersionDontCare:
+		return field != ""
+	case VersionEqual:
+		return field == q.Value
+	case VersionGreater:
+		return field > q.Value
+	case VersionGreaterOrEqual:
+		return field >= q.Value
+	case VersionLess:
+		return field < q.Value
+	case VersionLessOrEqual:
+		return field <= q.Value
+	case VersionPatternMatch:
+		matched, err := filepath.Match(q.Value, field)
+		return err == nil && matched
+	case VersionRegexp:
+		if q.Regexp == nil {
+			q.Regexp = regexp.MustCompile(q.Value)
+		}
+		return q.Regexp.FindStringIndex(field) != nil
+
+	}
+	panic("unknown relation")
+}
+
+// Query runs iteration through list
+func (q *FieldQuery) Query(list PackageCatalog) (result *PackageList) {
+	result = list.Scan(q)
+	return
+}
+
+// Fast depends on the query
+func (q *FieldQuery) Fast(list PackageCatalog) bool {
+	return false
+}
+
+// String interface
+func (q *FieldQuery) String() string {
+	escape := func(val string) string {
+		if strings.IndexAny(val, "()|,!{} \t\n") != -1 {
+			return "'" + strings.Replace(strings.Replace(val, "\\", "\\\\", -1), "'", "\\'", -1) + "'"
+		}
+		return val
+	}
+
+	var op string
+	switch q.Relation {
+	case VersionEqual:
+		op = "="
+	case VersionGreater:
+		op = ">>"
+	case VersionLess:
+		op = "<<"
+	case VersionRegexp:
+		op = "~"
+	case VersionPatternMatch:
+		op = "%"
+	case VersionGreaterOrEqual:
+		op = ">="
+	case VersionLessOrEqual:
+		op = "<="
+	}
+	return fmt.Sprintf("%s (%s %s)", escape(q.Field), op, escape(q.Value))
+}
+
+// Matches on dependency condition
+func (q *DependencyQuery) Matches(pkg *Package) bool {
+	return pkg.MatchesDependency(q.Dep)
+}
+
+// Fast is always true for dependency query
+func (q *DependencyQuery) Fast(list PackageCatalog) bool {
+	return list.SearchSupported()
+}
+
+// Query runs PackageList.Search
+func (q *DependencyQuery) Query(list PackageCatalog) (result *PackageList) {
+	if q.Fast(list) {
+		result = NewPackageList()
+		for _, pkg := range list.Search(q.Dep, true) {
+			result.Add(pkg)
+		}
+	} else {
+		result = list.Scan(q)
+	}
+
+	return
+}
+
+// String interface
+func (q *DependencyQuery) String() string {
+	return q.Dep.String()
+}
+
+// Matches on specific properties
+func (q *PkgQuery) Matches(pkg *Package) bool {
+	return pkg.Name == q.Pkg && pkg.Version == q.Version && pkg.Architecture == q.Arch
+}
+
+// Fast is always true for package query
+func (q *PkgQuery) Fast(list PackageCatalog) bool {
+	return true
+}
+
+// Query looks up specific package
+func (q *PkgQuery) Query(list PackageCatalog) (result *PackageList) {
+	return list.SearchByKey(q.Arch, q.Pkg, q.Version)
+}
+
+// String interface
+func (q *PkgQuery) String() string {
+	return fmt.Sprintf("%s_%s_%s", q.Pkg, q.Version, q.Arch)
+}

deb/reflist.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -84,6 +84,14 @@ func (l *PackageRefList) ForEach(handler func([]byte) error) error {
 	return err
 }
 
+// Has checks whether package is part of reflist
+func (l *PackageRefList) Has(p *Package) bool {
+	key := p.Key("")
+
+	i := sort.Search(len(l.Refs), func(j int) bool { return bytes.Compare(l.Refs[j], key) >= 0 })
+	return i < len(l.Refs) && bytes.Compare(l.Refs[i], key) == 0
+}
+
 // Substract returns all packages in l that are not in r
 func (l *PackageRefList) Substract(r *PackageRefList) *PackageRefList {
 	result := &PackageRefList{Refs: make([][]byte, 0, 128)}
@@ -194,34 +202,40 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle
 				}
 			}
 
-			// is pl & pr the same package, but different version?
-			if pl.Name == pr.Name && pl.Architecture == pr.Architecture {
-				result = append(result, PackageDiff{Left: pl, Right: pr})
-				il++
-				ir++
-				pl, pr = nil, nil
-			} else {
-				// otherwise pl or pr is missing on one of the sides
-				if rel < 0 {
+			// otherwise pl or pr is missing on one of the sides
+			if rel < 0 {
+				// compaction: +(,A) -(B,) --> !(A,B)
+				if len(result) > 0 && result[len(result)-1].Left == nil && result[len(result)-1].Right.Name == pl.Name &&
+					result[len(result)-1].Right.Architecture == pl.Architecture {
+					result[len(result)-1] = PackageDiff{Left: pl, Right: result[len(result)-1].Right}
+				} else {
 					result = append(result, PackageDiff{Left: pl, Right: nil})
-					il++
-					pl = nil
+				}
+				il++
+				pl = nil
+			} else {
+				// compaction: -(A,) +(,B) --> !(A,B)
+				if len(result) > 0 && result[len(result)-1].Right == nil && result[len(result)-1].Left.Name == pr.Name &&
+					result[len(result)-1].Left.Architecture == pr.Architecture {
+					result[len(result)-1] = PackageDiff{Left: result[len(result)-1].Left, Right: pr}
 				} else {
 					result = append(result, PackageDiff{Left: nil, Right: pr})
-					ir++
-					pr = nil
 				}
+				ir++
+				pr = nil
 			}
-
 		}
 	}
 
 	return
 }
 
-// Merge merges reflist r into current reflist. If overrideMatching, merge replaces matching packages (by architecture/name)
-// with reference from r, otherwise all packages are saved.
+// Merge merges reflist r into current reflist. If overrideMatching, merge
+// replaces matching packages (by architecture/name) with reference from r.
+// Otherwise, all packages are saved.
 func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result *PackageRefList) {
+	var overriddenArch, overridenName []byte
+
 	// pointer to left and right reflists
 	il, ir := 0, 0
 	// length of reflists
@@ -253,6 +267,8 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result
 			result.Refs = append(result.Refs, l.Refs[il])
 			il++
 			ir++
+			overridenName = nil
+			overriddenArch = nil
 		} else {
 			if overrideMatching {
 				partsL := bytes.Split(rl, []byte(" "))
@@ -261,11 +277,19 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result
 				partsR := bytes.Split(rr, []byte(" "))
 				archR, nameR := partsR[0][1:], partsR[1]
 
-				if bytes.Compare(archL, archR) == 0 && bytes.Compare(nameL, nameR) == 0 {
+				if bytes.Equal(archL, overriddenArch) && bytes.Equal(nameL, overridenName) {
+					// this package has already been overriden on the right
+					il++
+					continue
+				}
+
+				if bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) {
 					// override with package from the right
 					result.Refs = append(result.Refs, r.Refs[ir])
 					il++
 					ir++
+					overriddenArch = archL
+					overridenName = nameL
 					continue
 				}
 			}
@@ -277,10 +301,50 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result
 			} else {
 				result.Refs = append(result.Refs, r.Refs[ir])
 				ir++
+				overridenName = nil
+				overriddenArch = nil
 			}
-
 		}
 	}
 
 	return
 }
+
+// FilterLatestRefs takes in a reflist with potentially multiples of the same
+// packages and reduces it to only the latest of each package. The operations
+// are done in-place. This implements a "latest wins" approach which can be used
+// while merging two or more snapshots together.
+func FilterLatestRefs(r *PackageRefList) {
+	var (
+		lastArch, lastName, lastVer []byte
+		arch, name, ver             []byte
+		parts                       [][]byte
+	)
+
+	for i := 0; i < len(r.Refs); i++ {
+		parts = bytes.Split(r.Refs[i][1:], []byte(" "))
+		arch, name, ver = parts[0], parts[1], parts[2]
+
+		if bytes.Equal(arch, lastArch) && bytes.Equal(name, lastName) {
+			// Two packages are identical, check version and only one wins
+			vres := CompareVersions(string(ver), string(lastVer))
+
+			// Remove the older refs from the result
+			if vres > 0 {
+				// ver[i] > ver[i-1], remove element i-1
+				r.Refs = append(r.Refs[:i-1], r.Refs[i:]...)
+			} else {
+				// ver[i] < ver[i-1], remove element i
+				r.Refs = append(r.Refs[:i], r.Refs[i+1:]...)
+				arch, name, ver = lastArch, lastName, lastVer
+			}
+
+			// Compensate for the reduced set
+			i -= 1
+		}
+
+		lastArch, lastName, lastVer = arch, name, ver
+	}
+
+	return
+}

deb/reflist_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"errors"
@@ -14,6 +14,14 @@ type PackageRefListSuite struct {
 
 var _ = Suite(&PackageRefListSuite{})
 
+func toStrSlice(reflist *PackageRefList) (result []string) {
+	result = make([]string, reflist.Len())
+	for i, r := range reflist.Refs {
+		result[i] = string(r)
+	}
+	return
+}
+
 func (s *PackageRefListSuite) SetUpTest(c *C) {
 	s.list = NewPackageList()
 
@@ -23,7 +31,7 @@ func (s *PackageRefListSuite) SetUpTest(c *C) {
 	stanza["Package"] = "mars-invaders"
 	s.p3 = NewPackageFromControlFile(stanza)
 	stanza = packageStanza.Copy()
-	stanza["Size"] = "42"
+	stanza["Source"] = "unknown-planet"
 	s.p4 = NewPackageFromControlFile(stanza)
 	stanza = packageStanza.Copy()
 	stanza["Package"] = "lonely-strangers"
@@ -120,6 +128,19 @@ func (s *PackageRefListSuite) TestPackageRefListForeach(c *C) {
 	c.Check(err, Equals, e)
 }
 
+func (s *PackageRefListSuite) TestHas(c *C) {
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	reflist := NewPackageRefListFromPackageList(s.list)
+
+	c.Check(reflist.Has(s.p1), Equals, true)
+	c.Check(reflist.Has(s.p3), Equals, true)
+	c.Check(reflist.Has(s.p5), Equals, true)
+	c.Check(reflist.Has(s.p2), Equals, true)
+	c.Check(reflist.Has(s.p6), Equals, false)
+}
+
 func (s *PackageRefListSuite) TestSubstract(c *C) {
 	r1 := []byte("r1")
 	r2 := []byte("r2")
@@ -250,14 +271,6 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	reflistA := NewPackageRefListFromPackageList(listA)
 	reflistB := NewPackageRefListFromPackageList(listB)
 
-	toStrSlice := func(reflist *PackageRefList) (result []string) {
-		result = make([]string, reflist.Len())
-		for i, r := range reflist.Refs {
-			result[i] = string(r)
-		}
-		return
-	}
-
 	mergeAB := reflistA.Merge(reflistB, true)
 	mergeBA := reflistB.Merge(reflistA, true)
 
@@ -273,3 +286,32 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	c.Check(toStrSlice(mergeBAall), DeepEquals,
 		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"})
 }
+
+func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) {
+	packages := []*Package{
+		&Package{Name: "lib", Version: "1.0", Architecture: "i386"},
+		&Package{Name: "lib", Version: "1.2~bp1", Architecture: "i386"},
+		&Package{Name: "lib", Version: "1.2", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.2", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.3", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.3~bp2", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.5", Architecture: "i386"},
+		&Package{Name: "dpkg", Version: "1.6", Architecture: "i386"},
+	}
+
+	rl := NewPackageList()
+	rl.Add(packages[0])
+	rl.Add(packages[1])
+	rl.Add(packages[2])
+	rl.Add(packages[3])
+	rl.Add(packages[4])
+	rl.Add(packages[5])
+	rl.Add(packages[6])
+	rl.Add(packages[7])
+
+	result := NewPackageRefListFromPackageList(rl)
+	FilterLatestRefs(result)
+
+	c.Check(toStrSlice(result), DeepEquals,
+		[]string{"Pi386 dpkg 1.6", "Pi386 lib 1.2"})
+}

deb/remote.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -13,11 +13,19 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 )
 
+// RemoteRepo statuses
+const (
+	MirrorIdle = iota
+	MirrorUpdating
+)
+
 // RemoteRepo represents remote (fetchable) Debian repository.
 //
 // Repostitory could be filtered when fetching by components, architectures
@@ -36,21 +44,35 @@ type RemoteRepo struct {
 	Architectures []string
 	// Should we download sources?
 	DownloadSources bool
+	// Should we download .udebs?
+	DownloadUdebs bool
 	// Meta-information about repository
 	Meta Stanza
 	// Last update date
 	LastDownloadDate time.Time
 	// Checksums for release files
 	ReleaseFiles map[string]utils.ChecksumInfo
+	// Filter for packages
+	Filter string
+	// FilterWithDeps to include dependencies from filter query
+	FilterWithDeps bool
+	// Status marks state of repository (being updated, no action)
+	Status int
+	// WorkerPID is PID of the process modifying the mirror (if any)
+	WorkerPID int
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
+	// Temporary list of package refs
+	tempPackageRefs *PackageRefList
 	// Parsed archived root
 	archiveRootURL *url.URL
+	// Current list of packages (filled while updating mirror)
+	packageList *PackageList
 }
 
 // NewRemoteRepo creates new instance of Debian remote repository with specified params
 func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string,
-	architectures []string, downloadSources bool) (*RemoteRepo, error) {
+	architectures []string, downloadSources bool, downloadUdebs bool) (*RemoteRepo, error) {
 	result := &RemoteRepo{
 		UUID:            uuid.New(),
 		Name:            name,
@@ -59,6 +81,7 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone
 		Components:      components,
 		Architectures:   architectures,
 		DownloadSources: downloadSources,
+		DownloadUdebs:   downloadUdebs,
 	}
 
 	err := result.prepare()
@@ -66,13 +89,18 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone
 		return nil, err
 	}
 
-	if result.Distribution == "." || result.Distribution == "./" {
+	if strings.HasSuffix(result.Distribution, "/") || strings.HasPrefix(result.Distribution, ".") {
 		// flat repo
-		result.Distribution = ""
+		if !strings.HasPrefix(result.Distribution, ".") {
+			result.Distribution = "./" + result.Distribution
+		}
 		result.Architectures = nil
 		if len(result.Components) > 0 {
 			return nil, fmt.Errorf("components aren't supported for flat repos")
 		}
+		if result.DownloadUdebs {
+			return nil, fmt.Errorf("debian-installer udebs aren't supported for flat repos")
+		}
 		result.Components = nil
 	}
 
@@ -95,7 +123,10 @@ func (repo *RemoteRepo) prepare() error {
 func (repo *RemoteRepo) String() string {
 	srcFlag := ""
 	if repo.DownloadSources {
-		srcFlag = " [src]"
+		srcFlag += " [src]"
+	}
+	if repo.DownloadUdebs {
+		srcFlag += " [udeb]"
 	}
 	distribution := repo.Distribution
 	if distribution == "" {
@@ -106,7 +137,9 @@ func (repo *RemoteRepo) String() string {
 
 // IsFlat determines if repository is flat
 func (repo *RemoteRepo) IsFlat() bool {
-	return repo.Distribution == ""
+	// aptly < 0.5.1 had Distribution = "" for flat repos
+	// aptly >= 0.5.1 had Distribution = "./[path]/" for flat repos
+	return repo.Distribution == "" || (strings.HasPrefix(repo.Distribution, ".") && strings.HasSuffix(repo.Distribution, "/"))
 }
 
 // NumPackages return number of packages retrived from remote repo
@@ -122,6 +155,37 @@ func (repo *RemoteRepo) RefList() *PackageRefList {
 	return repo.packageRefs
 }
 
+// MarkAsUpdating puts current PID and sets status to updating
+func (repo *RemoteRepo) MarkAsUpdating() {
+	repo.Status = MirrorUpdating
+	repo.WorkerPID = os.Getpid()
+}
+
+// MarkAsIdle clears updating flag
+func (repo *RemoteRepo) MarkAsIdle() {
+	repo.Status = MirrorIdle
+	repo.WorkerPID = 0
+}
+
+// CheckLock returns error if mirror is being updated by another process
+func (repo *RemoteRepo) CheckLock() error {
+	if repo.Status == MirrorIdle || repo.WorkerPID == 0 {
+		return nil
+	}
+
+	p, err := os.FindProcess(repo.WorkerPID)
+	if err != nil {
+		return nil
+	}
+
+	err = p.Signal(syscall.Signal(0))
+	if err == nil {
+		return fmt.Errorf("mirror is locked by update operation, PID %d", repo.WorkerPID)
+	}
+
+	return nil
+}
+
 // ReleaseURL returns URL to Release* files in repo root
 func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
 	var path *url.URL
@@ -129,7 +193,7 @@ func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
 	if !repo.IsFlat() {
 		path = &url.URL{Path: fmt.Sprintf("dists/%s/%s", repo.Distribution, name)}
 	} else {
-		path = &url.URL{Path: name}
+		path = &url.URL{Path: filepath.Join(repo.Distribution, name)}
 	}
 
 	return repo.archiveRootURL.ResolveReference(path)
@@ -137,13 +201,13 @@ func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
 
 // FlatBinaryURL returns URL to Packages files for flat repo
 func (repo *RemoteRepo) FlatBinaryURL() *url.URL {
-	path := &url.URL{Path: "Packages"}
+	path := &url.URL{Path: filepath.Join(repo.Distribution, "Packages")}
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
 // FlatSourcesURL returns URL to Sources files for flat repo
 func (repo *RemoteRepo) FlatSourcesURL() *url.URL {
-	path := &url.URL{Path: "Sources"}
+	path := &url.URL{Path: filepath.Join(repo.Distribution, "Sources")}
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
@@ -160,6 +224,13 @@ func (repo *RemoteRepo) SourcesURL(component string) *url.URL {
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
+// UdebURL returns URL of Packages files for given component and
+// architecture
+func (repo *RemoteRepo) UdebURL(component string, architecture string) *url.URL {
+	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/debian-installer/binary-%s/Packages", repo.Distribution, component, architecture)}
+	return repo.archiveRootURL.ResolveReference(path)
+}
+
 // PackageURL returns URL of package file relative to repository root
 // architecture
 func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
@@ -170,8 +241,8 @@ func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
 // Fetch updates information about repository
 func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error {
 	var (
-		release *os.File
-		err     error
+		release, inrelease, releasesig *os.File
+		err                            error
 	)
 
 	if verifier == nil {
@@ -182,7 +253,7 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error
 		}
 	} else {
 		// 1. try InRelease file
-		inrelease, err := http.DownloadTemp(d, repo.ReleaseURL("InRelease").String())
+		inrelease, err = http.DownloadTemp(d, repo.ReleaseURL("InRelease").String())
 		if err != nil {
 			goto splitsignature
 		}
@@ -209,7 +280,7 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error
 			return err
 		}
 
-		releasesig, err := http.DownloadTemp(d, repo.ReleaseURL("Release.gpg").String())
+		releasesig, err = http.DownloadTemp(d, repo.ReleaseURL("Release.gpg").String())
 		if err != nil {
 			return err
 		}
@@ -275,7 +346,8 @@ ok:
 				return fmt.Errorf("unparseable hash sum line: %#v", line)
 			}
 
-			size, err := strconv.ParseInt(parts[1], 10, 64)
+			var size int64
+			size, err = strconv.ParseInt(parts[1], 10, 64)
 			if err != nil {
 				return fmt.Errorf("unable to parse size: %s", err)
 			}
@@ -313,11 +385,13 @@ ok:
 	return nil
 }
 
-// Download downloads all repo files
-func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, packageCollection *PackageCollection, packagePool aptly.PackagePool, ignoreMismatch bool) error {
-	list := NewPackageList()
-
-	progress.Printf("Downloading & parsing package files...\n")
+// DownloadPackageIndexes downloads & parses package index files
+func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
+	ignoreMismatch bool) error {
+	if repo.packageList != nil {
+		panic("packageList != nil")
+	}
+	repo.packageList = NewPackageList()
 
 	// Download and parse all Packages & Source files
 	packagesURLs := [][]string{}
@@ -331,6 +405,9 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, pa
 		for _, component := range repo.Components {
 			for _, architecture := range repo.Architectures {
 				packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
+				if repo.DownloadUdebs {
+					packagesURLs = append(packagesURLs, []string{repo.UdebURL(component, architecture).String(), "udeb"})
+				}
 			}
 			if repo.DownloadSources {
 				packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
@@ -367,18 +444,20 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, pa
 
 			if kind == "binary" {
 				p = NewPackageFromControlFile(stanza)
+			} else if kind == "udeb" {
+				p = NewUdebPackageFromControlFile(stanza)
 			} else if kind == "source" {
 				p, err = NewSourcePackageFromControlFile(stanza)
 				if err != nil {
 					return err
 				}
 			}
-			err = list.Add(p)
+			err = repo.packageList.Add(p)
 			if err != nil {
 				return err
 			}
 
-			err = packageCollection.Update(p)
+			err = collectionFactory.PackageCollection().Update(p)
 			if err != nil {
 				return err
 			}
@@ -387,74 +466,63 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, pa
 		progress.ShutdownBar()
 	}
 
-	progress.Printf("Building download queue...\n")
+	return nil
+}
 
-	// Build download queue
-	queued := make(map[string]PackageDownloadTask, list.Len())
-	count := 0
-	downloadSize := int64(0)
+// ApplyFilter applies filtering to already built PackageList
+func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQuery) (oldLen, newLen int, err error) {
+	repo.packageList.PrepareIndex()
 
-	err := list.ForEach(func(p *Package) error {
-		list, err := p.DownloadList(packagePool)
-		if err != nil {
-			return err
+	emptyList := NewPackageList()
+	emptyList.PrepareIndex()
+
+	oldLen = repo.packageList.Len()
+	repo.packageList, err = repo.packageList.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures)
+	if repo.packageList != nil {
+		newLen = repo.packageList.Len()
+	}
+	return
+}
+
+// BuildDownloadQueue builds queue, discards current PackageList
+func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool) (queue []PackageDownloadTask, downloadSize int64, err error) {
+	queue = make([]PackageDownloadTask, 0, repo.packageList.Len())
+	seen := make(map[string]struct{}, repo.packageList.Len())
+
+	err = repo.packageList.ForEach(func(p *Package) error {
+		list, err2 := p.DownloadList(packagePool)
+		if err2 != nil {
+			return err2
 		}
 		p.files = nil
 
 		for _, task := range list {
 			key := task.RepoURI + "-" + task.DestinationPath
-			_, found := queued[key]
+			_, found := seen[key]
 			if !found {
-				count++
+				queue = append(queue, task)
 				downloadSize += task.Checksums.Size
-				queued[key] = task
+				seen[key] = struct{}{}
 			}
 		}
 
 		return nil
 	})
 	if err != nil {
-		return fmt.Errorf("unable to build download queue: %s", err)
+		return
 	}
 
-	repo.packageRefs = NewPackageRefListFromPackageList(list)
+	repo.tempPackageRefs = NewPackageRefListFromPackageList(repo.packageList)
 	// free up package list, we don't need it after this point
-	list = nil
+	repo.packageList = nil
 
-	progress.Printf("Download queue: %d items, %.2f GiB size\n", count, float64(downloadSize)/(1024.0*1024.0*1024.0))
-
-	progress.InitBar(downloadSize, true)
-
-	// Download all package files
-	ch := make(chan error, len(queued))
-
-	for _, task := range queued {
-		d.DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
-	}
-
-	// We don't need queued after this point
-	queued = nil
-
-	// Wait for all downloads to finish
-	errors := make([]string, 0)
-
-	for count > 0 {
-		err = <-ch
-		if err != nil {
-			errors = append(errors, err.Error())
-		}
-		count--
-	}
-
-	progress.ShutdownBar()
-
-	if len(errors) > 0 {
-		return fmt.Errorf("download errors:\n  %s\n", strings.Join(errors, "\n  "))
-	}
+	return
+}
 
+// FinalizeDownload swaps for final value of package refs
+func (repo *RemoteRepo) FinalizeDownload() {
 	repo.LastDownloadDate = time.Now()
-
-	return nil
+	repo.packageRefs = repo.tempPackageRefs
 }
 
 // Encode does msgpack encoding of RemoteRepo
@@ -472,7 +540,43 @@ func (repo *RemoteRepo) Decode(input []byte) error {
 	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
 	err := decoder.Decode(repo)
 	if err != nil {
-		return err
+		if strings.HasPrefix(err.Error(), "codec.decoder: readContainerLen: Unrecognized descriptor byte: hex: 80") {
+			// probably it is broken DB from go < 1.2, try decoding w/o time.Time
+			var repo11 struct {
+				UUID             string
+				Name             string
+				ArchiveRoot      string
+				Distribution     string
+				Components       []string
+				Architectures    []string
+				DownloadSources  bool
+				Meta             Stanza
+				LastDownloadDate []byte
+				ReleaseFiles     map[string]utils.ChecksumInfo
+				Filter           string
+				FilterWithDeps   bool
+			}
+
+			decoder = codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
+			err2 := decoder.Decode(&repo11)
+			if err2 != nil {
+				return err
+			}
+
+			repo.UUID = repo11.UUID
+			repo.Name = repo11.Name
+			repo.ArchiveRoot = repo11.ArchiveRoot
+			repo.Distribution = repo11.Distribution
+			repo.Components = repo11.Components
+			repo.Architectures = repo11.Architectures
+			repo.DownloadSources = repo11.DownloadSources
+			repo.Meta = repo11.Meta
+			repo.ReleaseFiles = repo11.ReleaseFiles
+			repo.Filter = repo11.Filter
+			repo.FilterWithDeps = repo11.FilterWithDeps
+		} else {
+			return err
+		}
 	}
 	return repo.prepare()
 }

deb/remote_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"errors"
@@ -12,6 +12,7 @@ import (
 	"io/ioutil"
 	. "launchpad.net/gocheck"
 	"os"
+	"sort"
 )
 
 type NullVerifier struct {
@@ -72,19 +73,19 @@ type RemoteRepoSuite struct {
 	downloader        *http.FakeDownloader
 	progress          aptly.Progress
 	db                database.Storage
-	packageCollection *PackageCollection
+	collectionFactory *CollectionFactory
 	packagePool       aptly.PackagePool
 }
 
 var _ = Suite(&RemoteRepoSuite{})
 
 func (s *RemoteRepoSuite) SetUpTest(c *C) {
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false)
-	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false)
+	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false)
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress()
 	s.db, _ = database.OpenDB(c.MkDir())
-	s.packageCollection = NewPackageCollection(s.db)
+	s.collectionFactory = NewCollectionFactory(s.db)
 	s.packagePool = files.NewPackagePool(c.MkDir())
 	s.SetUpPackages()
 	s.progress.Start()
@@ -96,16 +97,21 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
-	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false)
+	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(err, ErrorMatches, ".*hexadecimal escape in host.*")
 }
 
 func (s *RemoteRepoSuite) TestFlatCreation(c *C) {
-	c.Check(s.flat.Distribution, Equals, "")
+	c.Check(s.flat.IsFlat(), Equals, true)
+	c.Check(s.flat.Distribution, Equals, "./")
 	c.Check(s.flat.Architectures, IsNil)
 	c.Check(s.flat.Components, IsNil)
 
-	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false)
+	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false)
+	c.Check(flat2.IsFlat(), Equals, true)
+	c.Check(flat2.Distribution, Equals, "./binary/")
+
+	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false)
 	c.Check(err, ErrorMatches, "components aren't supported for flat repos")
 }
 
@@ -114,8 +120,9 @@ func (s *RemoteRepoSuite) TestString(c *C) {
 	c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./")
 
 	s.repo.DownloadSources = true
+	s.repo.DownloadUdebs = true
 	s.flat.DownloadSources = true
-	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src]")
+	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb]")
 	c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./ [src]")
 }
 
@@ -146,6 +153,10 @@ func (s *RemoteRepoSuite) TestBinaryURL(c *C) {
 	c.Assert(s.repo.BinaryURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/binary-amd64/Packages")
 }
 
+func (s *RemoteRepoSuite) TestUdebURL(c *C) {
+	c.Assert(s.repo.UdebURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/debian-installer/binary-amd64/Packages")
+}
+
 func (s *RemoteRepoSuite) TestSourcesURL(c *C) {
 	c.Assert(s.repo.SourcesURL("main").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources")
 }
@@ -204,13 +215,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*")
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "component xyz not available in repo.*")
 }
@@ -244,20 +255,22 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", errors.New("HTTP 404"))
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", errors.New("HTTP 404"))
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
-	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
 
-	err = s.repo.Download(s.progress, s.downloader, s.packageCollection, s.packagePool, false)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.repo.FinalizeDownload()
 	c.Assert(s.repo.packageRefs, NotNil)
 
-	pkg, err := s.packageCollection.ByKey(s.repo.packageRefs.Refs[0])
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	result, err := pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "amanda-client")
 }
 
@@ -274,32 +287,35 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", errors.New("HTTP 404"))
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", errors.New("HTTP 404"))
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
-	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
-	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc", "abc")
-	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz", "abcd")
-	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz", "abcde")
 
-	err = s.repo.Download(s.progress, s.downloader, s.packageCollection, s.packagePool, false)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	q := make([]string, 4)
+	for i := range q {
+		q[i] = queue[i].RepoURI
+	}
+	sort.Strings(q)
+	c.Check(q[3], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+	c.Check(q[1], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc")
+	c.Check(q[2], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz")
+	c.Check(q[0], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz")
+
+	s.repo.FinalizeDownload()
 	c.Assert(s.repo.packageRefs, NotNil)
 
-	pkg, err := s.packageCollection.ByKey(s.repo.packageRefs.Refs[0])
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	result, err := pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "amanda-client")
 
-	pkg, err = s.packageCollection.ByKey(s.repo.packageRefs.Refs[1])
+	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
-
-	result, err = pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
 }
 
@@ -309,23 +325,25 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", errors.New("HTTP 404"))
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", errors.New("HTTP 404"))
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
-	downloader.ExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
 
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.Download(s.progress, downloader, s.packageCollection, s.packagePool, false)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.flat.FinalizeDownload()
 	c.Assert(s.flat.packageRefs, NotNil)
 
-	pkg, err := s.packageCollection.ByKey(s.flat.packageRefs.Refs[0])
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	result, err := pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "amanda-client")
 }
 
@@ -340,35 +358,39 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", errors.New("HTTP 404"))
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", errors.New("HTTP 404"))
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
-	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
-	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc", "abc")
-	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz", "abcd")
-	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz", "abcde")
 
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.Download(s.progress, downloader, s.packageCollection, s.packagePool, false)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	q := make([]string, 4)
+	for i := range q {
+		q[i] = queue[i].RepoURI
+	}
+	sort.Strings(q)
+	c.Check(q[3], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+	c.Check(q[1], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc")
+	c.Check(q[2], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz")
+	c.Check(q[0], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz")
+
+	s.flat.FinalizeDownload()
 	c.Assert(s.flat.packageRefs, NotNil)
 
-	pkg, err := s.packageCollection.ByKey(s.flat.packageRefs.Refs[0])
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	result, err := pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "amanda-client")
 
-	pkg, err = s.packageCollection.ByKey(s.flat.packageRefs.Refs[1])
+	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
 
-	result, err = pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
 }
 
@@ -394,7 +416,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 	r, err := s.collection.ByName("yandex")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
@@ -412,7 +434,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 	r, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 
 	r, err = s.collection.ByUUID(repo.UUID)
@@ -421,7 +443,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection := NewRemoteRepoCollection(s.db)
@@ -442,7 +464,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	s.collection.Add(repo)
 
 	count := 0
@@ -464,10 +486,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestDrop(c *C) {
-	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	s.collection.Add(repo1)
 
-	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false)
+	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false)
 	s.collection.Add(repo2)
 
 	r1, _ := s.collection.ByUUID(repo1.UUID)

deb/snapshot.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -9,6 +9,7 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/ugorji/go/codec"
 	"log"
+	"strings"
 	"time"
 )
 
@@ -125,7 +126,36 @@ func (s *Snapshot) Encode() []byte {
 // Decode decodes msgpack representation into Snapshot
 func (s *Snapshot) Decode(input []byte) error {
 	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
-	return decoder.Decode(s)
+	err := decoder.Decode(s)
+	if err != nil {
+		if strings.HasPrefix(err.Error(), "codec.decoder: readContainerLen: Unrecognized descriptor byte: hex: 80") {
+			// probably it is broken DB from go < 1.2, try decoding w/o time.Time
+			var snapshot11 struct {
+				UUID string
+				Name string
+				CreatedAt []byte
+
+				SourceKind string
+				SourceIDs  []string
+				Description string
+			}
+
+			decoder = codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
+			err2 := decoder.Decode(&snapshot11)
+			if err2 != nil {
+				return err
+			}
+
+			s.UUID = snapshot11.UUID
+			s.Name = snapshot11.Name
+			s.SourceKind = snapshot11.SourceKind
+			s.SourceIDs = snapshot11.SourceIDs
+			s.Description = snapshot11.Description
+		} else {
+			return err
+		}
+	}
+	return nil
 }
 
 // SnapshotCollection does listing, updating/adding/deleting of Snapshots
@@ -178,7 +208,10 @@ func (collection *SnapshotCollection) Update(snapshot *Snapshot) error {
 	if err != nil {
 		return err
 	}
-	return collection.db.Put(snapshot.RefKey(), snapshot.packageRefs.Encode())
+	if snapshot.packageRefs != nil {
+		return collection.db.Put(snapshot.RefKey(), snapshot.packageRefs.Encode())
+	}
+	return nil
 }
 
 // LoadComplete loads additional information about snapshot

deb/snapshot_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"errors"
@@ -15,7 +15,7 @@ var _ = Suite(&SnapshotSuite{})
 
 func (s *SnapshotSuite) SetUpTest(c *C) {
 	s.SetUpPackages()
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	s.repo.packageRefs = s.reflist
 }
 
@@ -108,11 +108,11 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
 	s.collection = NewSnapshotCollection(s.db)
 	s.SetUpPackages()
 
-	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	s.repo1.packageRefs = s.reflist
 	s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1)
 
-	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false)
+	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false)
 	s.repo2.packageRefs = s.reflist
 	s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2)
 
@@ -192,7 +192,7 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) {
 	c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1})
 	c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2})
 
-	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false)
+	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false)
 
 	c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot{})
 }

deb/version.go

@@ -1,7 +1,8 @@
-package debian
+package deb
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 	"unicode"
@@ -178,6 +179,8 @@ const (
 	VersionEqual
 	VersionGreaterOrEqual
 	VersionGreater
+	VersionPatternMatch
+	VersionRegexp
 )
 
 // Dependency is a parsed version of Debian dependency to package
@@ -186,6 +189,7 @@ type Dependency struct {
 	Relation     int
 	Version      string
 	Architecture string
+	Regexp       *regexp.Regexp
 }
 
 // Hash calculates some predefined unique ID of Dependency
@@ -207,6 +211,10 @@ func (d *Dependency) String() string {
 		rel = ">="
 	case VersionLessOrEqual:
 		rel = "<="
+	case VersionPatternMatch:
+		rel = "%"
+	case VersionRegexp:
+		rel = "~"
 	case VersionDontCare:
 		return fmt.Sprintf("%s [%s]", d.Pkg, d.Architecture)
 	}

deb/version_test.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	. "launchpad.net/gocheck"

debian/debian.go

@@ -1,2 +0,0 @@
-// Package debian implements Debian-specific repository handling
-package debian

debian/publish.go

@@ -1,467 +0,0 @@
-package debian
-
-import (
-	"bufio"
-	"bytes"
-	"code.google.com/p/go-uuid/uuid"
-	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/utils"
-	"github.com/ugorji/go/codec"
-	"log"
-	"path/filepath"
-	"sort"
-	"strings"
-	"time"
-)
-
-// PublishedRepo is a published for http/ftp representation of snapshot as Debian repository
-type PublishedRepo struct {
-	// Internal unique ID
-	UUID string
-	// Prefix & distribution should be unique across all published repositories
-	Prefix       string
-	Distribution string
-	Component    string
-	// Architectures is a list of all architectures published
-	Architectures []string
-	// Snapshot as a source of publishing
-	SnapshotUUID string
-
-	snapshot *Snapshot
-}
-
-// NewPublishedRepo creates new published repository
-func NewPublishedRepo(prefix string, distribution string, component string, architectures []string, snapshot *Snapshot) (*PublishedRepo, error) {
-	prefix = filepath.Clean(prefix)
-	if strings.HasPrefix(prefix, "/") {
-		prefix = prefix[1:]
-	}
-	if strings.HasSuffix(prefix, "/") {
-		prefix = prefix[:len(prefix)-1]
-	}
-	prefix = filepath.Clean(prefix)
-
-	for _, component := range strings.Split(prefix, "/") {
-		if component == ".." || component == "dists" || component == "pool" {
-			return nil, fmt.Errorf("invalid prefix %s", prefix)
-		}
-	}
-
-	return &PublishedRepo{
-		UUID:          uuid.New(),
-		Prefix:        prefix,
-		Distribution:  distribution,
-		Component:     component,
-		Architectures: architectures,
-		SnapshotUUID:  snapshot.UUID,
-		snapshot:      snapshot,
-	}, nil
-}
-
-// String returns human-readable represenation of PublishedRepo
-func (p *PublishedRepo) String() string {
-	var archs string
-
-	if len(p.Architectures) > 0 {
-		archs = fmt.Sprintf(" [%s]", strings.Join(p.Architectures, ", "))
-	}
-
-	return fmt.Sprintf("%s/%s (%s)%s publishes %s", p.Prefix, p.Distribution, p.Component, archs, p.snapshot.String())
-}
-
-// Key returns unique key identifying PublishedRepo
-func (p *PublishedRepo) Key() []byte {
-	return []byte("U" + p.Prefix + ">>" + p.Distribution)
-}
-
-// Encode does msgpack encoding of PublishedRepo
-func (p *PublishedRepo) Encode() []byte {
-	var buf bytes.Buffer
-
-	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
-	encoder.Encode(p)
-
-	return buf.Bytes()
-}
-
-// Decode decodes msgpack representation into PublishedRepo
-func (p *PublishedRepo) Decode(input []byte) error {
-	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
-	return decoder.Decode(p)
-}
-
-// Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
-func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage aptly.PublishedStorage, packageCollection *PackageCollection, signer utils.Signer, progress aptly.Progress) error {
-	err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
-	if err != nil {
-		return err
-	}
-	basePath := filepath.Join(p.Prefix, "dists", p.Distribution)
-	err = publishedStorage.MkDir(basePath)
-	if err != nil {
-		return err
-	}
-
-	if progress != nil {
-		progress.Printf("Loading packages...\n")
-	}
-
-	// Load all packages
-	list, err := NewPackageListFromRefList(p.snapshot.RefList(), packageCollection, progress)
-	if err != nil {
-		return fmt.Errorf("unable to load packages: %s", err)
-	}
-
-	if list.Len() == 0 {
-		return fmt.Errorf("snapshot is empty")
-	}
-
-	if len(p.Architectures) == 0 {
-		p.Architectures = list.Architectures(true)
-	}
-
-	if len(p.Architectures) == 0 {
-		return fmt.Errorf("unable to figure out list of architectures, please supply explicit list")
-	}
-
-	sort.Strings(p.Architectures)
-
-	generatedFiles := map[string]utils.ChecksumInfo{}
-
-	if progress != nil {
-		progress.Printf("Generating metadata files and linking package files...\n")
-	}
-
-	// For all architectures, generate release file
-	for _, arch := range p.Architectures {
-		if progress != nil {
-			progress.InitBar(int64(list.Len()), false)
-		}
-
-		var relativePath string
-		if arch == "source" {
-			relativePath = filepath.Join(p.Component, "source", "Sources")
-		} else {
-			relativePath = filepath.Join(p.Component, fmt.Sprintf("binary-%s", arch), "Packages")
-		}
-		err = publishedStorage.MkDir(filepath.Dir(filepath.Join(basePath, relativePath)))
-		if err != nil {
-			return err
-		}
-
-		packagesFile, err := publishedStorage.CreateFile(filepath.Join(basePath, relativePath))
-		if err != nil {
-			return fmt.Errorf("unable to creates Packages file: %s", err)
-		}
-
-		bufWriter := bufio.NewWriter(packagesFile)
-
-		err = list.ForEach(func(pkg *Package) error {
-			if progress != nil {
-				progress.AddBar(1)
-			}
-			if pkg.MatchesArchitecture(arch) {
-				err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, p.Component)
-				if err != nil {
-					return err
-				}
-
-				err = pkg.Stanza().WriteTo(bufWriter)
-				if err != nil {
-					return err
-				}
-				err = bufWriter.WriteByte('\n')
-				if err != nil {
-					return err
-				}
-
-				pkg.files = nil
-				pkg.deps = nil
-				pkg.extra = nil
-
-			}
-
-			return nil
-		})
-
-		if err != nil {
-			return fmt.Errorf("unable to process packages: %s", err)
-		}
-
-		err = bufWriter.Flush()
-		if err != nil {
-			return fmt.Errorf("unable to write Packages file: %s", err)
-		}
-
-		err = utils.CompressFile(packagesFile)
-		if err != nil {
-			return fmt.Errorf("unable to compress Packages files: %s", err)
-		}
-
-		packagesFile.Close()
-
-		checksumInfo, err := publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath))
-		if err != nil {
-			return fmt.Errorf("unable to collect checksums: %s", err)
-		}
-		generatedFiles[relativePath] = checksumInfo
-
-		checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+".gz"))
-		if err != nil {
-			return fmt.Errorf("unable to collect checksums: %s", err)
-		}
-		generatedFiles[relativePath+".gz"] = checksumInfo
-
-		checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+".bz2"))
-		if err != nil {
-			return fmt.Errorf("unable to collect checksums: %s", err)
-		}
-		generatedFiles[relativePath+".bz2"] = checksumInfo
-
-		if progress != nil {
-			progress.ShutdownBar()
-		}
-	}
-
-	release := make(Stanza)
-	release["Origin"] = p.Prefix + " " + p.Distribution
-	release["Label"] = p.Prefix + " " + p.Distribution
-	release["Codename"] = p.Distribution
-	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
-	release["Components"] = p.Component
-	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{"source"}), " ")
-	release["Description"] = " Generated by aptly\n"
-	release["MD5Sum"] = "\n"
-	release["SHA1"] = "\n"
-	release["SHA256"] = "\n"
-
-	for path, info := range generatedFiles {
-		release["MD5Sum"] += fmt.Sprintf(" %s %8d %s\n", info.MD5, info.Size, path)
-		release["SHA1"] += fmt.Sprintf(" %s %8d %s\n", info.SHA1, info.Size, path)
-		release["SHA256"] += fmt.Sprintf(" %s %8d %s\n", info.SHA256, info.Size, path)
-	}
-
-	releaseFile, err := publishedStorage.CreateFile(filepath.Join(basePath, "Release"))
-	if err != nil {
-		return fmt.Errorf("unable to create Release file: %s", err)
-	}
-
-	bufWriter := bufio.NewWriter(releaseFile)
-
-	err = release.WriteTo(bufWriter)
-	if err != nil {
-		return fmt.Errorf("unable to create Release file: %s", err)
-	}
-
-	err = bufWriter.Flush()
-	if err != nil {
-		return fmt.Errorf("unable to create Release file: %s", err)
-	}
-
-	releaseFilename := releaseFile.Name()
-	releaseFile.Close()
-
-	// Signing files might output to console, so flush progress writer first
-	if progress != nil {
-		progress.Flush()
-	}
-
-	if signer != nil {
-		err = signer.DetachedSign(releaseFilename, releaseFilename+".gpg")
-		if err != nil {
-			return fmt.Errorf("unable to sign Release file: %s", err)
-		}
-
-		err = signer.ClearSign(releaseFilename, filepath.Join(filepath.Dir(releaseFilename), "InRelease"))
-		if err != nil {
-			return fmt.Errorf("unable to sign Release file: %s", err)
-		}
-	}
-
-	return nil
-}
-
-// RemoveFiles removes files that were created by Publish
-//
-// It can remove prefix fully, and part of pool (for specific component)
-func (p *PublishedRepo) RemoveFiles(publishedStorage aptly.PublishedStorage, removePrefix, removePoolComponent bool) error {
-	if removePrefix {
-		err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists"))
-		if err != nil {
-			return err
-		}
-
-		return publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool"))
-	}
-
-	err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution))
-	if err != nil {
-		return err
-	}
-
-	if removePoolComponent {
-		err = publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool", p.Component))
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// PublishedRepoCollection does listing, updating/adding/deleting of PublishedRepos
-type PublishedRepoCollection struct {
-	db   database.Storage
-	list []*PublishedRepo
-}
-
-// NewPublishedRepoCollection loads PublishedRepos from DB and makes up collection
-func NewPublishedRepoCollection(db database.Storage) *PublishedRepoCollection {
-	result := &PublishedRepoCollection{
-		db: db,
-	}
-
-	blobs := db.FetchByPrefix([]byte("U"))
-	result.list = make([]*PublishedRepo, 0, len(blobs))
-
-	for _, blob := range blobs {
-		r := &PublishedRepo{}
-		if err := r.Decode(blob); err != nil {
-			log.Printf("Error decoding published repo: %s\n", err)
-		} else {
-			result.list = append(result.list, r)
-		}
-	}
-
-	return result
-}
-
-// Add appends new repo to collection and saves it
-func (collection *PublishedRepoCollection) Add(repo *PublishedRepo) error {
-	if collection.CheckDuplicate(repo) != nil {
-		return fmt.Errorf("published repo with prefix/distribution %s/%s already exists", repo.Prefix, repo.Distribution)
-	}
-
-	err := collection.Update(repo)
-	if err != nil {
-		return err
-	}
-
-	collection.list = append(collection.list, repo)
-	return nil
-}
-
-// CheckDuplicate verifies that there's no published repo with the same name
-func (collection *PublishedRepoCollection) CheckDuplicate(repo *PublishedRepo) *PublishedRepo {
-	for _, r := range collection.list {
-		if r.Prefix == repo.Prefix && r.Distribution == repo.Distribution {
-			return r
-		}
-	}
-
-	return nil
-}
-
-// Update stores updated information about repo in DB
-func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) error {
-	err := collection.db.Put(repo.Key(), repo.Encode())
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// LoadComplete loads additional information for remote repo
-func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, snapshotCollection *SnapshotCollection) error {
-	snapshot, err := snapshotCollection.ByUUID(repo.SnapshotUUID)
-	if err != nil {
-		return err
-	}
-
-	repo.snapshot = snapshot
-	return nil
-}
-
-// ByPrefixDistribution looks up repository by prefix & distribution
-func (collection *PublishedRepoCollection) ByPrefixDistribution(prefix, distribution string) (*PublishedRepo, error) {
-	for _, r := range collection.list {
-		if r.Prefix == prefix && r.Distribution == distribution {
-			return r, nil
-		}
-	}
-	return nil, fmt.Errorf("published repo with prefix/distribution %s/%s not found", prefix, distribution)
-}
-
-// ByUUID looks up repository by uuid
-func (collection *PublishedRepoCollection) ByUUID(uuid string) (*PublishedRepo, error) {
-	for _, r := range collection.list {
-		if r.UUID == uuid {
-			return r, nil
-		}
-	}
-	return nil, fmt.Errorf("published repo with uuid %s not found", uuid)
-}
-
-// BySnapshot looks up repository by snapshot source
-func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*PublishedRepo {
-	result := make([]*PublishedRepo, 0)
-	for _, r := range collection.list {
-		if r.SnapshotUUID == snapshot.UUID {
-			result = append(result, r)
-		}
-	}
-	return result
-}
-
-// ForEach runs method for each repository
-func (collection *PublishedRepoCollection) ForEach(handler func(*PublishedRepo) error) error {
-	var err error
-	for _, r := range collection.list {
-		err = handler(r)
-		if err != nil {
-			return err
-		}
-	}
-	return err
-}
-
-// Len returns number of remote repos
-func (collection *PublishedRepoCollection) Len() int {
-	return len(collection.list)
-}
-
-// Remove removes published repository, cleaning up directories, files
-func (collection *PublishedRepoCollection) Remove(publishedStorage aptly.PublishedStorage, prefix, distribution string) error {
-	repo, err := collection.ByPrefixDistribution(prefix, distribution)
-	if err != nil {
-		return err
-	}
-
-	removePrefix := true
-	removePoolComponent := true
-	repoPosition := -1
-
-	for i, r := range collection.list {
-		if r == repo {
-			repoPosition = i
-			continue
-		}
-		if r.Prefix == repo.Prefix {
-			removePrefix = false
-			if r.Component == repo.Component {
-				removePoolComponent = false
-			}
-		}
-	}
-
-	err = repo.RemoveFiles(publishedStorage, removePrefix, removePoolComponent)
-	if err != nil {
-		return err
-	}
-
-	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
-		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
-
-	return collection.db.Delete(repo.Key())
-}

debian/publish_test.go

@@ -1,486 +0,0 @@
-package debian
-
-import (
-	"errors"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/files"
-	. "launchpad.net/gocheck"
-	"os"
-	"path/filepath"
-)
-
-type pathExistsChecker struct {
-	*CheckerInfo
-}
-
-var PathExists = &pathExistsChecker{
-	&CheckerInfo{Name: "PathExists", Params: []string{"path"}},
-}
-
-func (checker *pathExistsChecker) Check(params []interface{}, names []string) (result bool, error string) {
-	_, err := os.Stat(params[0].(string))
-	return err == nil, ""
-}
-
-type NullSigner struct{}
-
-func (n *NullSigner) Init() error {
-	return nil
-}
-
-func (n *NullSigner) SetKey(keyRef string) {
-}
-
-func (n *NullSigner) SetKeyRing(keyring, secretKeyring string) {
-}
-
-func (n *NullSigner) DetachedSign(source string, destination string) error {
-	return nil
-}
-
-func (n *NullSigner) ClearSign(source string, destination string) error {
-	return nil
-}
-
-type PublishedRepoSuite struct {
-	PackageListMixinSuite
-	repo              *PublishedRepo
-	root              string
-	publishedStorage  aptly.PublishedStorage
-	packagePool       aptly.PackagePool
-	snapshot          *Snapshot
-	db                database.Storage
-	packageCollection *PackageCollection
-}
-
-var _ = Suite(&PublishedRepoSuite{})
-
-func (s *PublishedRepoSuite) SetUpTest(c *C) {
-	s.SetUpPackages()
-
-	s.db, _ = database.OpenDB(c.MkDir())
-
-	s.root = c.MkDir()
-	s.publishedStorage = files.NewPublishedStorage(s.root)
-	s.packagePool = files.NewPackagePool(s.root)
-
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
-	repo.packageRefs = s.reflist
-
-	s.snapshot, _ = NewSnapshotFromRepository("snap", repo)
-
-	s.repo, _ = NewPublishedRepo("ppa", "squeeze", "main", nil, s.snapshot)
-
-	s.packageCollection = NewPackageCollection(s.db)
-	s.packageCollection.Update(s.p1)
-	s.packageCollection.Update(s.p2)
-	s.packageCollection.Update(s.p3)
-
-	poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
-	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
-	f, err := os.Create(poolPath)
-	c.Assert(err, IsNil)
-	f.Close()
-}
-
-func (s *PublishedRepoSuite) TearDownTest(c *C) {
-	s.db.Close()
-}
-
-func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
-
-	for _, t := range []struct {
-		prefix        string
-		expected      string
-		errorExpected string
-	}{
-		{
-			prefix:   "ppa",
-			expected: "ppa",
-		},
-		{
-			prefix:   "",
-			expected: ".",
-		},
-		{
-			prefix:   "/",
-			expected: ".",
-		},
-		{
-			prefix:   "//",
-			expected: ".",
-		},
-		{
-			prefix:   "//ppa/",
-			expected: "ppa",
-		},
-		{
-			prefix:   "ppa/..",
-			expected: ".",
-		},
-		{
-			prefix:   "ppa/ubuntu/",
-			expected: "ppa/ubuntu",
-		},
-		{
-			prefix:   "ppa/../ubuntu/",
-			expected: "ubuntu",
-		},
-		{
-			prefix:        "../ppa/",
-			errorExpected: "invalid prefix .*",
-		},
-		{
-			prefix:        "../ppa/../ppa/",
-			errorExpected: "invalid prefix .*",
-		},
-		{
-			prefix:        "ppa/dists",
-			errorExpected: "invalid prefix .*",
-		},
-		{
-			prefix:        "ppa/pool",
-			errorExpected: "invalid prefix .*",
-		},
-	} {
-		repo, err := NewPublishedRepo(t.prefix, "squeeze", "main", nil, s.snapshot)
-		if t.errorExpected != "" {
-			c.Check(err, ErrorMatches, t.errorExpected)
-		} else {
-			c.Check(repo.Prefix, Equals, t.expected)
-		}
-	}
-}
-
-func (s *PublishedRepoSuite) TestPublish(c *C) {
-	err := s.repo.Publish(s.packagePool, s.publishedStorage, s.packageCollection, &NullSigner{}, nil)
-	c.Assert(err, IsNil)
-
-	c.Check(s.repo.Architectures, DeepEquals, []string{"i386"})
-
-	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
-	c.Assert(err, IsNil)
-
-	cfr := NewControlFileReader(rf)
-	st, err := cfr.ReadStanza()
-	c.Assert(err, IsNil)
-
-	c.Check(st["Origin"], Equals, "ppa squeeze")
-	c.Check(st["Components"], Equals, "main")
-	c.Check(st["Architectures"], Equals, "i386")
-
-	pf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages"))
-	c.Assert(err, IsNil)
-
-	cfr = NewControlFileReader(pf)
-
-	for i := 0; i < 3; i++ {
-		st, err = cfr.ReadStanza()
-		c.Assert(err, IsNil)
-
-		c.Check(st["Filename"], Equals, "pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
-	}
-
-	st, err = cfr.ReadStanza()
-	c.Assert(err, IsNil)
-	c.Assert(st, IsNil)
-
-	_, err = os.Stat(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb"))
-	c.Assert(err, IsNil)
-}
-
-func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
-	err := s.repo.Publish(s.packagePool, s.publishedStorage, s.packageCollection, nil, nil)
-	c.Assert(err, IsNil)
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists)
-}
-
-func (s *PublishedRepoSuite) TestString(c *C) {
-	c.Check(s.repo.String(), Equals,
-		"ppa/squeeze (main) publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
-	repo, _ := NewPublishedRepo("", "squeeze", "main", nil, s.snapshot)
-	c.Check(repo.String(), Equals,
-		"./squeeze (main) publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
-	repo, _ = NewPublishedRepo("", "squeeze", "main", []string{"i386", "amd64"}, s.snapshot)
-	c.Check(repo.String(), Equals,
-		"./squeeze (main) [i386, amd64] publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
-}
-
-func (s *PublishedRepoSuite) TestKey(c *C) {
-	c.Check(s.repo.Key(), DeepEquals, []byte("Uppa>>squeeze"))
-}
-
-func (s *PublishedRepoSuite) TestEncodeDecode(c *C) {
-	encoded := s.repo.Encode()
-	repo := &PublishedRepo{}
-	err := repo.Decode(encoded)
-
-	s.repo.snapshot = nil
-	c.Assert(err, IsNil)
-	c.Assert(repo, DeepEquals, s.repo)
-}
-
-type PublishedRepoCollectionSuite struct {
-	PackageListMixinSuite
-	db                  database.Storage
-	snapshotCollection  *SnapshotCollection
-	collection          *PublishedRepoCollection
-	snap1, snap2        *Snapshot
-	repo1, repo2, repo3 *PublishedRepo
-}
-
-var _ = Suite(&PublishedRepoCollectionSuite{})
-
-func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
-
-	s.snapshotCollection = NewSnapshotCollection(s.db)
-
-	s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
-	s.snap2 = NewSnapshotFromPackageList("snap2", []*Snapshot{}, NewPackageList(), "desc2")
-
-	s.snapshotCollection.Add(s.snap1)
-	s.snapshotCollection.Add(s.snap2)
-
-	s.repo1, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap1)
-	s.repo2, _ = NewPublishedRepo("", "anaconda", "main", []string{}, s.snap2)
-	s.repo3, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap2)
-
-	s.collection = NewPublishedRepoCollection(s.db)
-}
-
-func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) {
-	s.db.Close()
-}
-
-func (s *PublishedRepoCollectionSuite) TestAddByPrefixDistribution(c *C) {
-	r, err := s.collection.ByPrefixDistribution("ppa", "anaconda")
-	c.Assert(err, ErrorMatches, "*.not found")
-
-	c.Assert(s.collection.Add(s.repo1), IsNil)
-	c.Assert(s.collection.Add(s.repo1), ErrorMatches, ".*already exists")
-	c.Assert(s.collection.CheckDuplicate(s.repo2), IsNil)
-	c.Assert(s.collection.Add(s.repo2), IsNil)
-	c.Assert(s.collection.Add(s.repo3), ErrorMatches, ".*already exists")
-	c.Assert(s.collection.CheckDuplicate(s.repo3), Equals, s.repo1)
-
-	r, err = s.collection.ByPrefixDistribution("ppa", "anaconda")
-	c.Assert(err, IsNil)
-
-	err = s.collection.LoadComplete(r, s.snapshotCollection)
-	c.Assert(err, IsNil)
-	c.Assert(r.String(), Equals, s.repo1.String())
-
-	collection := NewPublishedRepoCollection(s.db)
-	r, err = collection.ByPrefixDistribution("ppa", "anaconda")
-	c.Assert(err, IsNil)
-
-	err = s.collection.LoadComplete(r, s.snapshotCollection)
-	c.Assert(err, IsNil)
-	c.Assert(r.String(), Equals, s.repo1.String())
-}
-
-func (s *PublishedRepoCollectionSuite) TestByUUID(c *C) {
-	r, err := s.collection.ByUUID(s.repo1.UUID)
-	c.Assert(err, ErrorMatches, "*.not found")
-
-	c.Assert(s.collection.Add(s.repo1), IsNil)
-
-	r, err = s.collection.ByUUID(s.repo1.UUID)
-	c.Assert(err, IsNil)
-
-	err = s.collection.LoadComplete(r, s.snapshotCollection)
-	c.Assert(err, IsNil)
-	c.Assert(r.String(), Equals, s.repo1.String())
-}
-
-func (s *PublishedRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
-	c.Assert(s.collection.Update(s.repo1), IsNil)
-
-	collection := NewPublishedRepoCollection(s.db)
-	r, err := collection.ByPrefixDistribution("ppa", "anaconda")
-	c.Assert(err, IsNil)
-	c.Assert(r.snapshot, IsNil)
-	c.Assert(s.collection.LoadComplete(r, s.snapshotCollection), IsNil)
-	c.Assert(r.snapshot.UUID, Equals, s.repo1.snapshot.UUID)
-}
-
-func (s *PublishedRepoCollectionSuite) TestForEachAndLen(c *C) {
-	s.collection.Add(s.repo1)
-
-	count := 0
-	err := s.collection.ForEach(func(*PublishedRepo) error {
-		count++
-		return nil
-	})
-	c.Assert(count, Equals, 1)
-	c.Assert(err, IsNil)
-
-	c.Check(s.collection.Len(), Equals, 1)
-
-	e := errors.New("c")
-
-	err = s.collection.ForEach(func(*PublishedRepo) error {
-		return e
-	})
-	c.Assert(err, Equals, e)
-}
-
-func (s *PublishedRepoCollectionSuite) TestBySnapshot(c *C) {
-	c.Check(s.collection.Add(s.repo1), IsNil)
-	c.Check(s.collection.Add(s.repo2), IsNil)
-
-	c.Check(s.collection.BySnapshot(s.snap1), DeepEquals, []*PublishedRepo{s.repo1})
-	c.Check(s.collection.BySnapshot(s.snap2), DeepEquals, []*PublishedRepo{s.repo2})
-}
-
-type PublishedRepoRemoveSuite struct {
-	PackageListMixinSuite
-	db                         database.Storage
-	snapshotCollection         *SnapshotCollection
-	collection                 *PublishedRepoCollection
-	root                       string
-	publishedStorage           aptly.PublishedStorage
-	snap1                      *Snapshot
-	repo1, repo2, repo3, repo4 *PublishedRepo
-}
-
-var _ = Suite(&PublishedRepoRemoveSuite{})
-
-func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
-
-	s.snapshotCollection = NewSnapshotCollection(s.db)
-
-	s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
-
-	s.snapshotCollection.Add(s.snap1)
-
-	s.repo1, _ = NewPublishedRepo("ppa", "anaconda", "main", []string{}, s.snap1)
-	s.repo2, _ = NewPublishedRepo("", "anaconda", "main", []string{}, s.snap1)
-	s.repo3, _ = NewPublishedRepo("ppa", "meduza", "main", []string{}, s.snap1)
-	s.repo4, _ = NewPublishedRepo("ppa", "osminog", "contrib", []string{}, s.snap1)
-
-	s.collection = NewPublishedRepoCollection(s.db)
-	s.collection.Add(s.repo1)
-	s.collection.Add(s.repo2)
-	s.collection.Add(s.repo3)
-	s.collection.Add(s.repo4)
-
-	s.root = c.MkDir()
-	s.publishedStorage = files.NewPublishedStorage(s.root)
-	s.publishedStorage.MkDir("ppa/dists/anaconda")
-	s.publishedStorage.MkDir("ppa/dists/meduza")
-	s.publishedStorage.MkDir("ppa/dists/osminog")
-	s.publishedStorage.MkDir("ppa/pool/main")
-	s.publishedStorage.MkDir("ppa/pool/contrib")
-	s.publishedStorage.MkDir("dists/anaconda")
-	s.publishedStorage.MkDir("pool/main")
-}
-
-func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
-	s.db.Close()
-}
-
-func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
-	s.repo1.RemoveFiles(s.publishedStorage, false, false)
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
-}
-
-func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
-	s.repo1.RemoveFiles(s.publishedStorage, false, true)
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
-}
-
-func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
-	s.repo1.RemoveFiles(s.publishedStorage, true, true)
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
-}
-
-func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
-	s.repo2.RemoveFiles(s.publishedStorage, true, true)
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), Not(PathExists))
-}
-
-func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
-	err := s.collection.Remove(s.publishedStorage, "ppa", "anaconda")
-	c.Check(err, IsNil)
-
-	_, err = s.collection.ByPrefixDistribution("ppa", "anaconda")
-	c.Check(err, ErrorMatches, ".*not found")
-
-	collection := NewPublishedRepoCollection(s.db)
-	_, err = collection.ByPrefixDistribution("ppa", "anaconda")
-	c.Check(err, ErrorMatches, ".*not found")
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
-
-	err = s.collection.Remove(s.publishedStorage, "ppa", "anaconda")
-	c.Check(err, ErrorMatches, ".*not found")
-
-	err = s.collection.Remove(s.publishedStorage, "ppa", "meduza")
-	c.Check(err, IsNil)
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
-}
-
-func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
-	err := s.collection.Remove(s.publishedStorage, ".", "anaconda")
-	c.Check(err, IsNil)
-
-	_, err = s.collection.ByPrefixDistribution(".", "anaconda")
-	c.Check(err, ErrorMatches, ".*not found")
-
-	collection := NewPublishedRepoCollection(s.db)
-	_, err = collection.ByPrefixDistribution(".", "anaconda")
-	c.Check(err, ErrorMatches, ".*not found")
-
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/"), Not(PathExists))
-	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/"), Not(PathExists))
-}

files/public.go

@@ -3,9 +3,10 @@ package files
 import (
 	"fmt"
 	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/utils"
+	"io"
 	"os"
 	"path/filepath"
+	"syscall"
 )
 
 // PublishedStorage abstract file system with public dirs (published repos)
@@ -13,9 +14,10 @@ type PublishedStorage struct {
 	rootPath string
 }
 
-// Check interface
+// Check interfaces
 var (
-	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+	_ aptly.PublishedStorage      = (*PublishedStorage)(nil)
+	_ aptly.LocalPublishedStorage = (*PublishedStorage)(nil)
 )
 
 // NewPublishedStorage creates new instance of PublishedStorage which specified root
@@ -33,51 +35,122 @@ func (storage *PublishedStorage) MkDir(path string) error {
 	return os.MkdirAll(filepath.Join(storage.rootPath, path), 0755)
 }
 
-// CreateFile creates file for writing under public path
-func (storage *PublishedStorage) CreateFile(path string) (*os.File, error) {
-	return os.Create(filepath.Join(storage.rootPath, path))
+// PutFile puts file into published storage at specified path
+func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
+	var (
+		source, f *os.File
+		err       error
+	)
+	source, err = os.Open(sourceFilename)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	f, err = os.Create(filepath.Join(storage.rootPath, path))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = io.Copy(f, source)
+	return err
+}
+
+// Remove removes single file under public path
+func (storage *PublishedStorage) Remove(path string) error {
+	filepath := filepath.Join(storage.rootPath, path)
+	return os.Remove(filepath)
 }
 
 // RemoveDirs removes directory structure under public path
-func (storage *PublishedStorage) RemoveDirs(path string) error {
+func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
 	filepath := filepath.Join(storage.rootPath, path)
-	fmt.Printf("Removing %s...\n", filepath)
+	if progress != nil {
+		progress.Printf("Removing %s...\n", filepath)
+	}
 	return os.RemoveAll(filepath)
 }
 
 // LinkFromPool links package file from pool to dist's pool location
 //
-// prefix is publishing prefix for this repo (e.g. empty or "ppa/")
-// component is component name when publishing (e.g. main)
-// poolDirectory is desired location in pool (like liba/libav/)
+// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
 // sourcePool is instance of aptly.PackagePool
 // sourcePath is filepath to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(prefix string, component string, poolDirectory string, sourcePool aptly.PackagePool, sourcePath string) (string, error) {
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool,
+	sourcePath, sourceMD5 string, force bool) error {
 	// verify that package pool is local pool is filesystem pool
 	_ = sourcePool.(*PackagePool)
 
 	baseName := filepath.Base(sourcePath)
-
-	relPath := filepath.Join("pool", component, poolDirectory, baseName)
-	poolPath := filepath.Join(storage.rootPath, prefix, "pool", component, poolDirectory)
+	poolPath := filepath.Join(storage.rootPath, publishedDirectory)
 
 	err := os.MkdirAll(poolPath, 0755)
 	if err != nil {
-		return "", err
+		return err
 	}
 
-	_, err = os.Stat(filepath.Join(poolPath, baseName))
-	if err == nil { // already exists, skip
-		return relPath, nil
+	var dstStat, srcStat os.FileInfo
+
+	dstStat, err = os.Stat(filepath.Join(poolPath, baseName))
+	if err == nil {
+		// already exists, check source file
+		srcStat, err = os.Stat(sourcePath)
+		if err != nil {
+			// source file doesn't exist? problem!
+			return err
+		}
+
+		srcSys := srcStat.Sys().(*syscall.Stat_t)
+		dstSys := dstStat.Sys().(*syscall.Stat_t)
+
+		// source and destination inodes match, no need to link
+		if srcSys.Ino == dstSys.Ino {
+			return nil
+		}
+
+		// source and destination have different inodes, if !forced, this is fatal error
+		if !force {
+			return fmt.Errorf("error linking file to %s: file already exists and is different", filepath.Join(poolPath, baseName))
+		}
+
+		// forced, so remove destination
+		err = os.Remove(filepath.Join(poolPath, baseName))
+		if err != nil {
+			return err
+		}
 	}
 
-	err = os.Link(sourcePath, filepath.Join(poolPath, baseName))
-	return relPath, err
+	// destination doesn't exist (or forced), create link
+	return os.Link(sourcePath, filepath.Join(poolPath, baseName))
 }
 
-// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
-func (storage *PublishedStorage) ChecksumsForFile(path string) (utils.ChecksumInfo, error) {
-	return utils.ChecksumsForFile(filepath.Join(storage.rootPath, path))
+// Filelist returns list of files under prefix
+func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
+	root := filepath.Join(storage.rootPath, prefix)
+	result := []string{}
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			result = append(result, path[len(root)+1:])
+		}
+		return nil
+	})
+
+	if err != nil && os.IsNotExist(err) {
+		// file path doesn't exist, consider it empty
+		return []string{}, nil
+	}
+
+	return result, err
+}
+
+// RenameFile renames (moves) file
+func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
+	return os.Rename(filepath.Join(storage.rootPath, oldName), filepath.Join(storage.rootPath, newName))
 }