Version bump to 0.9.5.

published repositories even if cleanup fails. #153

.travis.yml

@@ -1,15 +1,20 @@
 language: go
 
 go:
-  - 1.1
+  - 1.3.3
-  - 1.2
-  - 1.2.1
   - tip
 
 env:
   global:
     - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
-
+    - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
+    - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -y python-virtualenv graphviz
+  - virtualenv env
+  - . env/bin/activate
+  - pip install boto requests python-swiftclient
 install:
   - make prepare
 
@@ -19,3 +24,12 @@ script: make travis
 matrix:
     allow_failures:
         - go: tip
+
+
+notifications:
+  webhooks:
+    urls:
+      - "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
+    on_success: change  # options: [always|never|change] default: always
+    on_failure: always  # options: [always|never|change] default: always
+    on_start: false     # default: false

AUTHORS

@@ -0,0 +1,17 @@
+List of contributors, in chronological order:
+
+* Andrey Smirnov (https://github.com/smira)
+* Sebastien Binet (https://github.com/sbinet)
+* Ryan Uber (https://github.com/ryanuber)
+* Simon Aquino (https://github.com/queeno)
+* Vincent Batoufflet (https://github.com/vbatoufflet)
+* Ivan Kurnosov (https://github.com/zerkms)
+* Dmitrii Kashin (https://github.com/freehck)
+* Chris Read (https://github.com/cread)
+* Rohan Garg (https://github.com/shadeslayer)
+* Russ Allbery (https://github.com/rra)
+* Sylvain Baubeau (https://github.com/lebauce)
+* Andrea Bernardo Ciddio (https://github.com/bcandrea)
+* Michael Koval (https://github.com/mkoval)
+* Alexander Guy (https://github.com/alexanderguy)
+* Sebastien Badia (https://github.com/sbadia)

Gomfile

@@ -1,22 +1,33 @@
 gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
-gom 'code.google.com/p/go.crypto/ssh/terminal', :commit => '7aa593ce8cea'
+gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2'
-gom 'code.google.com/p/gographviz', :commit => '212766062629'
+gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556'
 gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
-gom 'github.com/cheggaaa/pb', :commit => '74be7a1388046f374ac36e93d46f5d56e856f827'
+gom 'github.com/AlekSi/pointer', :commit => '5f6d527dae3d678b46fbb20331ddf44e2b841943'
-gom 'github.com/gonuts/commander', :commit => 'f8ba4e959ca914268227c3ebbd7f6bf0bb35541a'
+gom 'github.com/cheggaaa/pb', :commit => '2c1b74620cc58a81ac152ee2d322e28c806d81ed'
-gom 'github.com/gonuts/flag', :commit => '741a6cbd37a30dedc93f817e7de6aaf0ca38a493'
+gom 'github.com/gin-gonic/gin', :commit => 'b1758d3bfa09e61ddbc1c9a627e936eec6a170de'
+gom 'github.com/jlaffaye/ftp', :commit => 'fec71e62e457557fbe85cefc847a048d57815d76'
+gom 'github.com/julienschmidt/httprouter', :commit => '46807412fe50aaceb73bb57061c2230fd26a1640'
+gom 'github.com/mattn/go-shellwords', :commit => 'c7ca6f94add751566a61cf2199e1de78d4c3eee4'
+gom 'github.com/mitchellh/goamz/s3', :commit => 'e7664b32019f31fd1bdf33f9e85f28722f700405'
 gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => '527a7b286bd095794af6c519627b7ed3d8fd067a'
+gom 'github.com/ncw/swift', :commit => '384ef27c70645e285f8bb9d02276bf654d06027e'
+gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
+gom 'github.com/smira/flag', :commit => '357ed3e599ffcbd4aeaa828e1d10da2df3ea5107'
+gom 'github.com/smira/go-ftp-protocol/protocol', :commit => '066b75c2b70dca7ae10b1b88b47534a3c31ccfaa'
+gom 'github.com/syndtr/gosnappy/snappy', :commit => 'ce8acff4829e0c2458a67ead32390ac0a381c862'
+gom 'github.com/syndtr/goleveldb/leveldb', :commit => '97e257099d2ab9578151ba85e2641e2cd14d3ca8'
 gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
+gom 'github.com/vaughan0/go-ini', :commit => 'a98ad7ee00ec53921f08832bc06ecf7fd600e6a1'
 gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
+gom 'golang.org/x/crypto/ssh/terminal', :commit => 'a7ead6ddf06233883deca151dffaef2effbf498f'
 
 group :test do
-    gom 'launchpad.net/gocheck'
+    gom 'gopkg.in/check.v1'
 end
 
 group :development do
    gom 'github.com/golang/lint/golint'
    gom 'github.com/mattn/goveralls'
    gom 'github.com/axw/gocov/gocov'
-   gom 'code.google.com/p/go.tools/cmd/cover'
+   gom 'golang.org/x/tools/cmd/cover'
 end

Makefile

@@ -1,6 +1,6 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-PACKAGES=database debian files http utils
+PACKAGES=context database deb files http query swift s3 utils
-ALL_PACKAGES=aptly cmd console database debian files http utils
+ALL_PACKAGES=api aptly context cmd console database deb files http query swift s3 utils
 BINPATH=$(abspath ./_vendor/bin)
 GOM_ENVIRONMENT=-test
 PYTHON?=python
@@ -36,16 +36,14 @@ coverage: coverage.out
 	rm -f coverage.out
 
 check:
-	$(GOM) exec go tool vet -all=true -shadow=true $(ALL_PACKAGES:%=./%)
+	$(GOM) exec go tool vet -all=true $(ALL_PACKAGES:%=./%)
 	$(GOM) exec golint $(ALL_PACKAGES:%=./%)
 
 install:
 	$(GOM) build -o $(BINPATH)/aptly
 
 system-test: install
-ifeq ($(GOVERSION),$(filter $(GOVERSION),go1.2 go1.2.1 devel))
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
-endif
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
 	PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long
 
@@ -61,4 +59,28 @@ mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-.PHONY: coverage.out
+package:
+	rm -rf root/
+	mkdir -p root/usr/bin/ root/usr/share/man/man1/ root/etc/bash_completion.d
+	cp $(BINPATH)/aptly root/usr/bin
+	cp man/aptly.1 root/usr/share/man/man1
+	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
+	gzip root/usr/share/man/man1/aptly.1
+	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
+	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 --deb-recommends graphviz -C root/ .
+	mv aptly_$(VERSION)_*.deb ~
+
+src-package:
+	rm -rf aptly-$(VERSION)
+	mkdir -p aptly-$(VERSION)/src/github.com/smira/aptly/
+	cd aptly-$(VERSION)/src/github.com/smira/ && git clone https://github.com/smira/aptly && cd aptly && git checkout v$(VERSION)
+	cd aptly-$(VERSION)/src/github.com/smira/aptly && gom -production install
+	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . \( -name .git -o -name .bzr -o -name .hg \) -print | xargs rm -rf
+	rm -rf aptly-$(VERSION)/src/github.com/smira/aptly/_vendor/{pkg,bin}
+	mkdir -p aptly-$(VERSION)/bash_completion.d
+	(cd aptly-$(VERSION)/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/$(VERSION)/aptly)
+	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
+	rm -rf aptly-$(VERSION)
+	curl -T aptly-$(VERSION)-src.tar.bz2 -usmira:$(BINTRAY_KEY) https://api.bintray.com/content/smira/aptly/aptly/$(VERSION)/$(VERSION)/aptly-$(VERSION)-src.tar.bz2
+
+.PHONY: coverage.out

README.rst

@@ -8,8 +8,17 @@ aptly
 .. image:: https://coveralls.io/repos/smira/aptly/badge.png?branch=HEAD
     :target: https://coveralls.io/r/smira/aptly?branch=HEAD
 
+.. image:: https://badges.gitter.im/Join Chat.svg
+    :target: https://gitter.im/smira/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+
+.. image:: http://goreportcard.com/badge/gojp/goreportcard
+    :target: http://goreportcard.com/report/gojp/goreportcard
+
 Aptly is a swiss army knife for Debian repository management.
 
+.. image:: http://www.aptly.info/img/aptly_logo.png
+    :target: http://www.aptly.info/
+
 Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support use
 mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
 
@@ -20,27 +29,48 @@ Aptly features: ("+" means planned features)
 * publish snapshot as Debian repository, ready to be consumed by apt
 * controlled update of one or more packages in snapshot from upstream mirror, tracking dependencies
 * merge two or more snapshots into one
-* filter repository by search query, pulling dependencies when required (+)
+* filter repository by search query, pulling dependencies when required
-* publish self-made packages as Debian repositories (+)
+* publish self-made packages as Debian repositories
+* REST API for remote access
 * mirror repositories "as-is" (without resigning with user's key) (+)
 * support for yum repositories (+)
 
 Current limitations:
 
-* debian-installer and translations not supported yet
+* translations are not supported yet
 
 Download
 --------
 
+To install aptly on Debian/Ubuntu, add new repository to /etc/apt/sources.list::
+
+    deb http://repo.aptly.info/ squeeze main
+
+And import key that is used to sign the release::
+
+    $ apt-key adv --keyserver keys.gnupg.net --recv-keys E083A3782A194991
+
+After that you can install aptly as any other software package::
+
+    $ apt-get update
+    $ apt-get install aptly
+
+Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
+Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
+
+If you would like to use nightly builds (unstable), please use following repository::
+
+    deb http://repo.aptly.info/ nightly main
+
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.1+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.3+ required)::
 
     go get -u github.com/mattn/gom
     mkdir -p $GOPATH/src/github.com/smira/aptly
     git clone https://github.com/smira/aptly $GOPATH/src/github.com/smira/aptly
     cd $GOPATH/src/github.com/smira/aptly
-    gom install
+    gom -production install
     gom build -o $GOPATH/bin/aptly
 
 Aptly is using `gom <https://github.com/mattn/gom>`_ to fix external dependencies, so regular ``go get github.com/smira/aptly``

api/api.go

@@ -0,0 +1,113 @@
+// Package api provides implementation of aptly REST API
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"sort"
+	"time"
+)
+
+// Lock order acquisition (canonical):
+//  1. RemoteRepoCollection
+//  2. LocalRepoCollection
+//  3. SnapshotCollection
+//  4. PublishedRepoCollection
+
+// GET /api/version
+func apiVersion(c *gin.Context) {
+	c.JSON(200, gin.H{"Version": aptly.Version})
+}
+
+// Periodically flushes CollectionFactory to free up memory used by collections,
+// flushing caches.
+//
+// Should be run in goroutine!
+func cacheFlusher() {
+	ticker := time.Tick(15 * time.Minute)
+
+	for {
+		<-ticker
+
+		// lock everything to eliminate in-progress calls
+		r := context.CollectionFactory().RemoteRepoCollection()
+		r.Lock()
+		defer r.Unlock()
+
+		l := context.CollectionFactory().LocalRepoCollection()
+		l.Lock()
+		defer l.Unlock()
+
+		s := context.CollectionFactory().SnapshotCollection()
+		s.Lock()
+		defer s.Unlock()
+
+		p := context.CollectionFactory().PublishedRepoCollection()
+		p.Lock()
+		defer p.Unlock()
+
+		// all collections locked, flush them
+		context.CollectionFactory().Flush()
+	}
+}
+
+// Common piece of code to show list of packages,
+// with searching & details if requested
+func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
+	result := []*deb.Package{}
+
+	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), nil)
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	queryS := c.Request.URL.Query().Get("q")
+	if queryS != "" {
+		q, err := query.Parse(c.Request.URL.Query().Get("q"))
+		if err != nil {
+			c.Fail(400, err)
+			return
+		}
+
+		withDeps := c.Request.URL.Query().Get("withDeps") == "1"
+		architecturesList := []string{}
+
+		if withDeps {
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = list.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				return
+			}
+		}
+
+		list.PrepareIndex()
+
+		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
+			nil, context.DependencyOptions(), architecturesList)
+		if err != nil {
+			c.Fail(500, fmt.Errorf("unable to search: %s", err))
+		}
+	}
+
+	if c.Request.URL.Query().Get("format") == "details" {
+		list.ForEach(func(p *deb.Package) error {
+			result = append(result, p)
+			return nil
+		})
+
+		c.JSON(200, result)
+	} else {
+		c.JSON(200, list.Strings())
+	}
+}

api/files.go

@@ -0,0 +1,185 @@
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func verifyPath(path string) bool {
+	path = filepath.Clean(path)
+	for _, part := range strings.Split(path, string(filepath.Separator)) {
+		if part == ".." || part == "." {
+			return false
+		}
+	}
+
+	return true
+
+}
+
+func verifyDir(c *gin.Context) bool {
+	if !verifyPath(c.Params.ByName("dir")) {
+		c.Fail(400, fmt.Errorf("wrong dir"))
+		return false
+	}
+
+	return true
+}
+
+// GET /files
+func apiFilesListDirs(c *gin.Context) {
+	list := []string{}
+
+	err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == context.UploadPath() {
+			return nil
+		}
+
+		if info.IsDir() {
+			list = append(list, filepath.Base(path))
+			return filepath.SkipDir
+		}
+
+		return nil
+	})
+
+	if err != nil && !os.IsNotExist(err) {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// POST /files/:dir/
+func apiFilesUpload(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	err := os.MkdirAll(path, 0777)
+
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	stored := []string{}
+
+	for _, files := range c.Request.MultipartForm.File {
+		for _, file := range files {
+			src, err := file.Open()
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+			defer src.Close()
+
+			destPath := filepath.Join(path, filepath.Base(file.Filename))
+			dst, err := os.Create(destPath)
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+			defer dst.Close()
+
+			_, err = io.Copy(dst, src)
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+
+			stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
+		}
+	}
+
+	c.JSON(200, stored)
+
+}
+
+// GET /files/:dir
+func apiFilesListFiles(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	list := []string{}
+	root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == root {
+			return nil
+		}
+
+		list = append(list, filepath.Base(path))
+
+		return nil
+	})
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			c.Fail(404, err)
+		} else {
+			c.Fail(500, err)
+		}
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// DELETE /files/:dir
+func apiFilesDeleteDir(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// DELETE /files/:dir/:name
+func apiFilesDeleteFile(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	if !verifyPath(c.Params.ByName("name")) {
+		c.Fail(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
+	if err != nil {
+		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
+			c.Fail(500, err)
+			return
+		}
+	}
+
+	c.JSON(200, gin.H{})
+}

api/graph.go

@@ -0,0 +1,75 @@
+package api
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/deb"
+	"io"
+	"mime"
+	"os"
+	"os/exec"
+)
+
+// GET /api/graph.:ext
+func apiGraph(c *gin.Context) {
+	var (
+		err    error
+		output []byte
+	)
+
+	ext := c.Params.ByName("ext")
+
+	factory := context.CollectionFactory()
+
+	factory.RemoteRepoCollection().RLock()
+	defer factory.RemoteRepoCollection().RUnlock()
+	factory.LocalRepoCollection().RLock()
+	defer factory.LocalRepoCollection().RUnlock()
+	factory.SnapshotCollection().RLock()
+	defer factory.SnapshotCollection().RUnlock()
+	factory.PublishedRepoCollection().RLock()
+	defer factory.PublishedRepoCollection().RUnlock()
+
+	graph, err := deb.BuildGraph(factory)
+	if err != nil {
+		c.JSON(500, err)
+		return
+	}
+
+	buf := bytes.NewBufferString(graph.String())
+
+	command := exec.Command("dot", "-T"+ext)
+	command.Stderr = os.Stderr
+
+	stdin, err := command.StdinPipe()
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	_, err = io.Copy(stdin, buf)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	err = stdin.Close()
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	output, err = command.Output()
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		return
+	}
+
+	mimeType := mime.TypeByExtension("." + ext)
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+
+	c.Data(200, mimeType, output)
+}

api/packages.go

@@ -0,0 +1,16 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/packages/:key
+func apiPackagesShow(c *gin.Context) {
+	p, err := context.CollectionFactory().PackageCollection().ByKey([]byte(c.Params.ByName("key")))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	c.JSON(200, p)
+}

api/publish.go

@@ -0,0 +1,338 @@
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
+	"strings"
+)
+
+// SigningOptions is a shared between publish API GPG options structure
+type SigningOptions struct {
+	Skip           bool
+	Batch          bool
+	GpgKey         string
+	Keyring        string
+	SecretKeyring  string
+	Passphrase     string
+	PassphraseFile string
+}
+
+func getSigner(options *SigningOptions) (utils.Signer, error) {
+	if options.Skip {
+		return nil, nil
+	}
+
+	signer := &utils.GpgSigner{}
+	signer.SetKey(options.GpgKey)
+	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
+	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
+	signer.SetBatch(options.Batch)
+
+	err := signer.Init()
+	if err != nil {
+		return nil, err
+	}
+
+	return signer, nil
+}
+
+// Replace '_' with '/' and double '__' with single '_'
+func parseEscapedPath(path string) string {
+	result := strings.Replace(strings.Replace(path, "_", "/", -1), "//", "_", -1)
+	if result == "" {
+		result = "."
+	}
+	return result
+}
+
+// GET /publish
+func apiPublishList(c *gin.Context) {
+	localCollection := context.CollectionFactory().LocalRepoCollection()
+	localCollection.RLock()
+	defer localCollection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.RLock()
+	defer snapshotCollection.RUnlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	result := make([]*deb.PublishedRepo, 0, collection.Len())
+
+	err := collection.ForEach(func(repo *deb.PublishedRepo) error {
+		err := collection.LoadComplete(repo, context.CollectionFactory())
+		if err != nil {
+			return err
+		}
+
+		result = append(result, repo)
+
+		return nil
+	})
+
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, result)
+}
+
+// POST /publish/:prefix
+func apiPublishRepoOrSnapshot(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+
+	var b struct {
+		SourceKind string `binding:"required"`
+		Sources    []struct {
+			Component string
+			Name      string `binding:"required"`
+		} `binding:"required"`
+		Distribution   string
+		Label          string
+		Origin         string
+		ForceOverwrite bool
+		Architectures  []string
+		Signing        SigningOptions
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	if len(b.Sources) == 0 {
+		c.Fail(400, fmt.Errorf("unable to publish: soures are empty"))
+		return
+	}
+
+	var components []string
+	var sources []interface{}
+
+	if b.SourceKind == "snapshot" {
+		var snapshot *deb.Snapshot
+
+		snapshotCollection := context.CollectionFactory().SnapshotCollection()
+		snapshotCollection.RLock()
+		defer snapshotCollection.RUnlock()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+
+			snapshot, err = snapshotCollection.ByName(source.Name)
+			if err != nil {
+				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			sources = append(sources, snapshot)
+		}
+	} else if b.SourceKind == "local" {
+		var localRepo *deb.LocalRepo
+
+		localCollection := context.CollectionFactory().LocalRepoCollection()
+		localCollection.RLock()
+		defer localCollection.RUnlock()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+
+			localRepo, err = localCollection.ByName(source.Name)
+			if err != nil {
+				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			err = localCollection.LoadComplete(localRepo)
+			if err != nil {
+				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+			}
+
+			sources = append(sources, localRepo)
+		}
+	} else {
+		c.Fail(400, fmt.Errorf("unknown SourceKind"))
+		return
+	}
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, context.CollectionFactory())
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		return
+	}
+	published.Origin = b.Origin
+	published.Label = b.Label
+
+	duplicate := collection.CheckDuplicate(published)
+	if duplicate != nil {
+		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
+		c.Fail(400, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate))
+		return
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		return
+	}
+
+	err = collection.Add(published)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+	}
+
+	c.JSON(201, published)
+}
+
+// PUT /publish/:prefix/:distribution
+func apiPublishUpdateSwitch(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	var b struct {
+		ForceOverwrite bool
+		Signing        SigningOptions
+		Snapshots      []struct {
+			Component string `binding:"required"`
+			Name      string `binding:"required"`
+		}
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	// published.LoadComplete would touch local repo collection
+	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
+	localRepoCollection.RLock()
+	defer localRepoCollection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.RLock()
+	defer snapshotCollection.RUnlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		c.Fail(404, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+	err = collection.LoadComplete(published, context.CollectionFactory())
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	var updatedComponents []string
+
+	if published.SourceKind == "local" {
+		if len(b.Snapshots) > 0 {
+			c.Fail(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
+			return
+		}
+		updatedComponents = published.Components()
+		for _, component := range updatedComponents {
+			published.UpdateLocalRepo(component)
+		}
+	} else if published.SourceKind == "snapshot" {
+		publishedComponents := published.Components()
+		for _, snapshotInfo := range b.Snapshots {
+			if !utils.StrSliceHasItem(publishedComponents, snapshotInfo.Component) {
+				c.Fail(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
+				return
+			}
+
+			snapshot, err := snapshotCollection.ByName(snapshotInfo.Name)
+			if err != nil {
+				c.Fail(404, err)
+				return
+			}
+
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+
+			published.UpdateSnapshot(snapshotInfo.Component, snapshot)
+			updatedComponents = append(updatedComponents, snapshotInfo.Component)
+		}
+	} else {
+		c.Fail(500, fmt.Errorf("unknown published repository type"))
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+	}
+
+	err = collection.Update(published)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+	}
+
+	err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
+		context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+	}
+
+	c.JSON(200, published)
+}
+
+// DELETE /publish/:prefix/:distribution
+func apiPublishDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	// published.LoadComplete would touch local repo collection
+	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
+	localRepoCollection.RLock()
+	defer localRepoCollection.RUnlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	err := collection.Remove(context, storage, prefix, distribution,
+		context.CollectionFactory(), context.Progress(), force)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to drop: %s", err))
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}

api/repos.go

@@ -0,0 +1,370 @@
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
+	"os"
+	"path/filepath"
+)
+
+// GET /api/repos
+func apiReposList(c *gin.Context) {
+	result := []*deb.LocalRepo{}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	context.CollectionFactory().LocalRepoCollection().ForEach(func(r *deb.LocalRepo) error {
+		result = append(result, r)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/repos
+func apiReposCreate(c *gin.Context) {
+	var b struct {
+		Name                string `binding:"required"`
+		Comment             string
+		DefaultDistribution string
+		DefaultComponent    string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	repo := deb.NewLocalRepo(b.Name, b.Comment)
+	repo.DefaultComponent = b.DefaultComponent
+	repo.DefaultDistribution = b.DefaultDistribution
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	err := context.CollectionFactory().LocalRepoCollection().Add(repo)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, repo)
+}
+
+// PUT /api/repos/:name
+func apiReposEdit(c *gin.Context) {
+	var b struct {
+		Comment             string
+		DefaultDistribution string
+		DefaultComponent    string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	if b.Comment != "" {
+		repo.Comment = b.Comment
+	}
+	if b.DefaultDistribution != "" {
+		repo.DefaultDistribution = b.DefaultDistribution
+	}
+	if b.DefaultComponent != "" {
+		repo.DefaultComponent = b.DefaultComponent
+	}
+
+	err = collection.Update(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
+// GET /api/repos/:name
+func apiReposShow(c *gin.Context) {
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
+// DELETE /api/repos/:name
+func apiReposDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.RLock()
+	defer snapshotCollection.RUnlock()
+
+	publishedCollection := context.CollectionFactory().PublishedRepoCollection()
+	publishedCollection.RLock()
+	defer publishedCollection.RUnlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	published := publishedCollection.ByLocalRepo(repo)
+	if len(published) > 0 {
+		c.Fail(409, fmt.Errorf("unable to drop, local repo is published"))
+		return
+	}
+
+	if !force {
+		snapshots := snapshotCollection.ByLocalRepoSource(repo)
+		if len(snapshots) > 0 {
+			c.Fail(409, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override"))
+			return
+		}
+	}
+
+	err = collection.Drop(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// GET /api/repos/:name/packages
+func apiReposPackagesShow(c *gin.Context) {
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	showPackages(c, repo.RefList())
+}
+
+// Handler for both add and delete
+func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p *deb.Package) error) {
+	var b struct {
+		PackageRefs []string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	// verify package refs and build package list
+	for _, ref := range b.PackageRefs {
+		var p *deb.Package
+
+		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
+		if err != nil {
+			if err == database.ErrNotFound {
+				c.Fail(404, fmt.Errorf("package %s: %s", ref, err))
+			} else {
+				c.Fail(500, err)
+			}
+			return
+		}
+		err = cb(list, p)
+		if err != nil {
+			c.Fail(400, err)
+			return
+		}
+	}
+
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save: %s", err))
+		return
+	}
+
+	c.JSON(200, repo)
+
+}
+
+// POST /repos/:name/packages
+func apiReposPackagesAdd(c *gin.Context) {
+	apiReposPackagesAddDelete(c, func(list *deb.PackageList, p *deb.Package) error {
+		return list.Add(p)
+	})
+}
+
+// DELETE /repos/:name/packages
+func apiReposPackagesDelete(c *gin.Context) {
+	apiReposPackagesAddDelete(c, func(list *deb.PackageList, p *deb.Package) error {
+		list.Remove(p)
+		return nil
+	})
+}
+
+// POST /repos/:name/file/:dir/:file
+func apiReposPackageFromFile(c *gin.Context) {
+	// redirect all work to dir method
+	apiReposPackageFromDir(c)
+}
+
+// POST /repos/:name/file/:dir
+func apiReposPackageFromDir(c *gin.Context) {
+	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
+	noRemove := c.Request.URL.Query().Get("noRemove") == "1"
+
+	if !verifyDir(c) {
+		return
+	}
+
+	fileParam := c.Params.ByName("file")
+	if fileParam != "" && !verifyPath(fileParam) {
+		c.Fail(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	verifier := &utils.GpgVerifier{}
+
+	var (
+		sources                      []string
+		packageFiles, failedFiles    []string
+		processedFiles, failedFiles2 []string
+		reporter                     = &aptly.RecordingResultReporter{
+			Warnings:     []string{},
+			AddedLines:   []string{},
+			RemovedLines: []string{},
+		}
+		list *deb.PackageList
+	)
+
+	if fileParam == "" {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"))}
+	} else {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
+	}
+
+	packageFiles, failedFiles, err = deb.CollectPackageFiles(sources, reporter)
+
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to collect package files: %s", err))
+		return
+	}
+
+	list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to load packages: %s", err))
+		return
+	}
+
+	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+		context.CollectionFactory().PackageCollection(), reporter)
+	failedFiles = append(failedFiles, failedFiles2...)
+
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to import package files: %s", err))
+		return
+	}
+
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save: %s", err))
+		return
+	}
+
+	if !noRemove {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err := os.Remove(file)
+			if err != nil {
+				reporter.Warning("unable to remove file %s: %s", file, err)
+			}
+		}
+
+		// atempt to remove dir, if it fails, that's fine: probably it's not empty
+		os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	}
+
+	if failedFiles == nil {
+		failedFiles = []string{}
+	}
+
+	c.JSON(200, gin.H{
+		"Report":      reporter,
+		"FailedFiles": failedFiles,
+	})
+}

api/router.go

@@ -0,0 +1,82 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+	ctx "github.com/smira/aptly/context"
+	"net/http"
+)
+
+var context *ctx.AptlyContext
+
+// Router returns prebuilt with routes http.Handler
+func Router(c *ctx.AptlyContext) http.Handler {
+	context = c
+
+	go cacheFlusher()
+
+	router := gin.Default()
+	router.Use(gin.ErrorLogger())
+
+	root := router.Group("/api")
+
+	{
+		root.GET("/version", apiVersion)
+	}
+
+	{
+		root.GET("/repos", apiReposList)
+		root.POST("/repos", apiReposCreate)
+		root.GET("/repos/:name", apiReposShow)
+		root.PUT("/repos/:name", apiReposEdit)
+		root.DELETE("/repos/:name", apiReposDrop)
+
+		root.GET("/repos/:name/packages", apiReposPackagesShow)
+		root.POST("/repos/:name/packages", apiReposPackagesAdd)
+		root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
+
+		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
+		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+
+		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
+	}
+
+	{
+		root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
+	}
+
+	{
+		root.GET("/files", apiFilesListDirs)
+		root.POST("/files/:dir", apiFilesUpload)
+		root.GET("/files/:dir", apiFilesListFiles)
+		root.DELETE("/files/:dir", apiFilesDeleteDir)
+		root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
+	}
+
+	{
+		root.GET("/publish", apiPublishList)
+		root.POST("/publish", apiPublishRepoOrSnapshot)
+		root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
+		root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
+		root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+	}
+
+	{
+		root.GET("/snapshots", apiSnapshotsList)
+		root.POST("/snapshots", apiSnapshotsCreate)
+		root.PUT("/snapshots/:name", apiSnapshotsUpdate)
+		root.GET("/snapshots/:name", apiSnapshotsShow)
+		root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
+		root.DELETE("/snapshots/:name", apiSnapshotsDrop)
+		root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+	}
+
+	{
+		root.GET("/packages/:key", apiPackagesShow)
+	}
+
+	{
+		root.GET("/graph.:ext", apiGraph)
+	}
+
+	return router
+}

api/snapshot.go

@@ -0,0 +1,410 @@
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/deb"
+)
+
+// GET /api/snapshots
+func apiSnapshotsList(c *gin.Context) {
+	SortMethodString := c.Request.URL.Query().Get("sort")
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	if SortMethodString == "" {
+		SortMethodString = "name"
+	}
+
+	result := []*deb.Snapshot{}
+	collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
+		result = append(result, snapshot)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/mirrors/:name/snapshots/
+func apiSnapshotsCreateFromMirror(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.RemoteRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().RemoteRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	repo, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		c.Fail(409, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// POST /api/snapshots
+func apiSnapshotsCreate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name            string `binding:"required"`
+		Description     string
+		SourceSnapshots []string
+		PackageRefs     []string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	if b.Description == "" {
+		if len(b.SourceSnapshots)+len(b.PackageRefs) == 0 {
+			b.Description = "Created as empty"
+		}
+	}
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	sources := make([]*deb.Snapshot, len(b.SourceSnapshots))
+
+	for i := range b.SourceSnapshots {
+		sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
+		if err != nil {
+			c.Fail(404, err)
+			return
+		}
+
+		err = snapshotCollection.LoadComplete(sources[i])
+		if err != nil {
+			c.Fail(500, err)
+			return
+		}
+	}
+
+	list := deb.NewPackageList()
+
+	// verify package refs and build package list
+	for _, ref := range b.PackageRefs {
+		var p *deb.Package
+
+		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
+		if err != nil {
+			if err == database.ErrNotFound {
+				c.Fail(404, fmt.Errorf("package %s: %s", ref, err))
+			} else {
+				c.Fail(500, err)
+			}
+			return
+		}
+		err = list.Add(p)
+		if err != nil {
+			c.Fail(400, err)
+			return
+		}
+	}
+
+	snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// POST /api/repos/:name/snapshots
+func apiSnapshotsCreateFromRepository(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.LocalRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	repo, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// PUT /api/snapshots/:name
+func apiSnapshotsUpdate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string
+		Description string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshot, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	_, err = collection.ByName(b.Name)
+	if err == nil {
+		c.Fail(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
+		return
+	}
+
+	if b.Name != "" {
+		snapshot.Name = b.Name
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, snapshot)
+}
+
+// GET /api/snapshots/:name
+func apiSnapshotsShow(c *gin.Context) {
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, snapshot)
+}
+
+// DELETE /api/snapshots/:name
+func apiSnapshotsDrop(c *gin.Context) {
+	name := c.Params.ByName("name")
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	publishedCollection := context.CollectionFactory().PublishedRepoCollection()
+	publishedCollection.RLock()
+	defer publishedCollection.RUnlock()
+
+	snapshot, err := snapshotCollection.ByName(name)
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	published := publishedCollection.BySnapshot(snapshot)
+
+	if len(published) > 0 {
+		c.Fail(409, fmt.Errorf("unable to drop: snapshot is published"))
+		return
+	}
+
+	if !force {
+		snapshots := snapshotCollection.BySnapshotSource(snapshot)
+		if len(snapshots) > 0 {
+			c.Fail(409, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override"))
+			return
+		}
+	}
+
+	err = snapshotCollection.Drop(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// GET /api/snapshots/:name/diff/:withSnapshot
+func apiSnapshotsDiff(c *gin.Context) {
+	onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1"
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshotA, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotA)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotB)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	// Calculate diff
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	result := []deb.PackageDiff{}
+
+	for _, pdiff := range diff {
+		if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) {
+			continue
+		}
+
+		result = append(result, pdiff)
+	}
+
+	c.JSON(200, result)
+}
+
+// GET /api/snapshots/:name/packages
+func apiSnapshotsSearchPackages(c *gin.Context) {
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	showPackages(c, snapshot.RefList())
+}

aptly/interfaces.go

@@ -5,7 +5,6 @@ package aptly
 import (
 	"github.com/smira/aptly/utils"
 	"io"
-	"os"
 )
 
 // PackagePool is asbtraction of package pool storage.
@@ -26,18 +25,32 @@ type PackagePool interface {
 
 // PublishedStorage is abstraction of filesystem storing all published repositories
 type PublishedStorage interface {
-	// PublicPath returns root of public part
-	PublicPath() string
 	// MkDir creates directory recursively under public path
 	MkDir(path string) error
-	// CreateFile creates file for writing under public path
+	// PutFile puts file into published storage at specified path
-	CreateFile(path string) (*os.File, error)
+	PutFile(path string, sourceFilename string) error
 	// RemoveDirs removes directory structure under public path
-	RemoveDirs(path string) error
+	RemoveDirs(path string, progress Progress) error
+	// Remove removes single file under public path
+	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(prefix string, component string, poolDirectory string, sourcePool PackagePool, sourcePath string) (string, error)
+	LinkFromPool(publishedDirectory string, sourcePool PackagePool, sourcePath, sourceMD5 string, force bool) error
-	// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
+	// Filelist returns list of files under prefix
-	ChecksumsForFile(path string) (utils.ChecksumInfo, error)
+	Filelist(prefix string) ([]string, error)
+	// RenameFile renames (moves) file
+	RenameFile(oldName, newName string) error
+}
+
+// LocalPublishedStorage is published storage on local filesystem
+type LocalPublishedStorage interface {
+	// PublicPath returns root of public part
+	PublicPath() string
+}
+
+// PublishedStorageProvider is a thing that returns PublishedStorage by name
+type PublishedStorageProvider interface {
+	// GetPublishedStorage returns PublishedStorage by name
+	GetPublishedStorage(name string) PublishedStorage
 }
 
 // Progress is a progress displaying entity, it allows progress bars & simple prints
@@ -77,6 +90,8 @@ type Downloader interface {
 	// Shutdown stops downloader after current tasks are finished,
 	// but doesn't process rest of queue
 	Shutdown()
+	// Abort stops downloader without waiting for shutdown
+	Abort()
 	// GetProgress returns Progress object
 	GetProgress() Progress
 }

aptly/report.go

@@ -0,0 +1,67 @@
+package aptly
+
+import (
+	"fmt"
+)
+
+// ResultReporter is abstraction for result reporting from complex processing functions
+type ResultReporter interface {
+	// Warning is non-fatal error message
+	Warning(msg string, a ...interface{})
+	// Removed is signal that something has been removed
+	Removed(msg string, a ...interface{})
+	// Added is signal that something has been added
+	Added(msg string, a ...interface{})
+}
+
+// ConsoleResultReporter is implementation of ResultReporter that prints in colors to console
+type ConsoleResultReporter struct {
+	Progress Progress
+}
+
+// Check interface
+var (
+	_ ResultReporter = &ConsoleResultReporter{}
+)
+
+// Warning is non-fatal error message (yellow)
+func (c *ConsoleResultReporter) Warning(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@y[!]@| @!"+msg+"@|", a...)
+}
+
+// Removed is signal that something has been removed (red)
+func (c *ConsoleResultReporter) Removed(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@r[-]@| "+msg, a...)
+}
+
+// Added is signal that something has been added (green)
+func (c *ConsoleResultReporter) Added(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@g[+]@| "+msg, a...)
+}
+
+// RecordingResultReporter is implementation of ResultReporter that collects all messages
+type RecordingResultReporter struct {
+	Warnings     []string
+	AddedLines   []string `json:"Added"`
+	RemovedLines []string `json:"Removed"`
+}
+
+// Check interface
+var (
+	_ ResultReporter = &RecordingResultReporter{}
+)
+
+// Warning is non-fatal error message
+func (r *RecordingResultReporter) Warning(msg string, a ...interface{}) {
+	r.Warnings = append(r.Warnings, fmt.Sprintf(msg, a...))
+}
+
+// Removed is signal that something has been removed
+func (r *RecordingResultReporter) Removed(msg string, a ...interface{}) {
+	r.RemovedLines = append(r.RemovedLines, fmt.Sprintf(msg, a...))
+}
+
+// Added is signal that something has been added
+func (r *RecordingResultReporter) Added(msg string, a ...interface{}) {
+	r.AddedLines = append(r.AddedLines, fmt.Sprintf(msg, a...))
+}

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
 // Version of aptly
-const Version = "0.4.1"
+const Version = "0.9.5"
 
 // Enable debugging features?
 const EnableDebug = false

cmd/api.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdAPI() *commander.Command {
+	return &commander.Command{
+		UsageLine: "api",
+		Short:     "start API server/issue requests",
+		Subcommands: []*commander.Command{
+			makeCmdAPIServe(),
+		},
+	}
+}

cmd/api_serve.go

@@ -0,0 +1,52 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/api"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"net/http"
+)
+
+func aptlyAPIServe(cmd *commander.Command, args []string) error {
+	var (
+		err error
+	)
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	listen := context.Flags().Lookup("listen").Value.String()
+
+	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
+
+	err = http.ListenAndServe(listen, api.Router(context))
+	if err != nil {
+		return fmt.Errorf("unable to serve: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdAPIServe() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyAPIServe,
+		UsageLine: "serve",
+		Short:     "start API HTTP service",
+		Long: `
+Stat HTTP server with aptly REST API.
+
+Example:
+
+  $ aptly api serve -listen=:8080
+`,
+		Flag: *flag.NewFlagSet("aptly-serve", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening")
+
+	return cmd
+
+}

cmd/cmd.go

@@ -3,28 +3,26 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
 	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"os"
 	"time"
 )
 
 // ListPackagesRefList shows list of packages in PackageRefList
-func ListPackagesRefList(reflist *debian.PackageRefList) (err error) {
+func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 	fmt.Printf("Packages:\n")
 
 	if reflist == nil {
 		return
 	}
 
-	packageCollection := debian.NewPackageCollection(context.database)
-
 	err = reflist.ForEach(func(key []byte) error {
-		p, err := packageCollection.ByKey(key)
+		p, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
-		if err != nil {
+		if err2 != nil {
-			return err
+			return err2
 		}
 		fmt.Printf("  %s\n", p)
 		return nil
@@ -36,6 +34,18 @@ func ListPackagesRefList(reflist *debian.PackageRefList) (err error) {
 	return
 }
 
+// LookupOption checks boolean flag with default (usually config) and command-line
+// setting
+func LookupOption(defaultValue bool, flags *flag.FlagSet, name string) (result bool) {
+	result = defaultValue
+
+	if flags.IsSet(name) {
+		result = flags.Lookup(name).Value.Get().(bool)
+	}
+
+	return
+}
+
 // RootCommand creates root command in command tree
 func RootCommand() *commander.Command {
 	cmd := &commander.Command{
@@ -47,22 +57,26 @@ repositories, manage local repositories, filter them, merge,
 upgrade individual packages, take snapshots and publish them
 back as Debian repositories.
 
-aptly goal is to establish repeatiblity and controlled changes
+aptly's goal is to establish repeatability and controlled changes
-in package environment. aptly allows to fix set of packages in
+in a package-centric environment. aptly allows one to fix a set of packages
-repository, so that package installation and upgrade becomes
+in a repository, so that package installation and upgrade becomes
-deterministic. At the same time aptly allows to perform controlled,
+deterministic. At the same time aptly allows one to perform controlled,
 fine-grained changes in repository contents to transition your
 package environment to new version.`,
 		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
 		Subcommands: []*commander.Command{
+			makeCmdConfig(),
 			makeCmdDb(),
 			makeCmdGraph(),
 			makeCmdMirror(),
 			makeCmdRepo(),
 			makeCmdServe(),
 			makeCmdSnapshot(),
+			makeCmdTask(),
 			makeCmdPublish(),
 			makeCmdVersion(),
+			makeCmdPackage(),
+			makeCmdAPI(),
 		},
 	}
 

cmd/config.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdConfig() *commander.Command {
+	return &commander.Command{
+		UsageLine: "config",
+		Short:     "manage aptly configuration",
+		Subcommands: []*commander.Command{
+			makeCmdConfigShow(),
+		},
+	}
+}

cmd/config_show.go

@@ -0,0 +1,38 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/smira/commander"
+)
+
+func aptlyConfigShow(cmd *commander.Command, args []string) error {
+
+	config := context.Config()
+	prettyJSON, err := json.MarshalIndent(config, "", "    ")
+
+	if err != nil {
+		return fmt.Errorf("unable to dump the config file: %s", err)
+	}
+
+	fmt.Println(string(prettyJSON))
+
+	return nil
+}
+
+func makeCmdConfigShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyConfigShow,
+		UsageLine: "show",
+		Short:     "show current aptly's config",
+		Long: `
+Command show displays the current aptly configuration.
+
+Example:
+
+  $ aptly config show
+
+`,
+	}
+	return cmd
+}

cmd/context.go

@@ -1,151 +1,31 @@
 package cmd
 
 import (
-	"fmt"
+	ctx "github.com/smira/aptly/context"
-	"github.com/gonuts/commander"
+	"github.com/smira/flag"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/console"
-	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/debian"
-	"github.com/smira/aptly/files"
-	"github.com/smira/aptly/http"
-	"github.com/smira/aptly/utils"
-	"os"
-	"path/filepath"
-	"runtime"
-	"runtime/pprof"
-	"strings"
-	"time"
 )
 
-// Common context shared by all commands
+var context *ctx.AptlyContext
-var context struct {
-	progress          aptly.Progress
-	downloader        aptly.Downloader
-	database          database.Storage
-	packagePool       aptly.PackagePool
-	publishedStorage  aptly.PublishedStorage
-	dependencyOptions int
-	architecturesList []string
-	// Debug features
-	fileCPUProfile *os.File
-	fileMemProfile *os.File
-	fileMemStats   *os.File
-}
-
-// InitContext initializes context with default settings
-func InitContext(cmd *commander.Command) error {
-	var err error
-
-	context.dependencyOptions = 0
-	if utils.Config.DepFollowSuggests || cmd.Flag.Lookup("dep-follow-suggests").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowSuggests
-	}
-	if utils.Config.DepFollowRecommends || cmd.Flag.Lookup("dep-follow-recommends").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowRecommends
-	}
-	if utils.Config.DepFollowAllVariants || cmd.Flag.Lookup("dep-follow-all-variants").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowAllVariants
-	}
-	if utils.Config.DepFollowSource || cmd.Flag.Lookup("dep-follow-source").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowSource
-	}
-
-	context.architecturesList = utils.Config.Architectures
-	optionArchitectures := cmd.Flag.Lookup("architectures").Value.String()
-	if optionArchitectures != "" {
-		context.architecturesList = strings.Split(optionArchitectures, ",")
-	}
-
-	context.progress = console.NewProgress()
-	context.progress.Start()
-
-	context.downloader = http.NewDownloader(utils.Config.DownloadConcurrency, context.progress)
-
-	context.database, err = database.OpenDB(filepath.Join(utils.Config.RootDir, "db"))
-	if err != nil {
-		return fmt.Errorf("can't open database: %s", err)
-	}
-
-	context.packagePool = files.NewPackagePool(utils.Config.RootDir)
-	context.publishedStorage = files.NewPublishedStorage(utils.Config.RootDir)
-
-	if aptly.EnableDebug {
-		cpuprofile := cmd.Flag.Lookup("cpuprofile").Value.String()
-		if cpuprofile != "" {
-			context.fileCPUProfile, err = os.Create(cpuprofile)
-			if err != nil {
-				return err
-			}
-			pprof.StartCPUProfile(context.fileCPUProfile)
-		}
-
-		memprofile := cmd.Flag.Lookup("memprofile").Value.String()
-		if memprofile != "" {
-			context.fileMemProfile, err = os.Create(memprofile)
-			if err != nil {
-				return err
-			}
-		}
-
-		memstats := cmd.Flag.Lookup("memstats").Value.String()
-		if memstats != "" {
-			interval := cmd.Flag.Lookup("meminterval").Value.Get().(time.Duration)
-
-			context.fileMemStats, err = os.Create(memstats)
-			if err != nil {
-				return err
-			}
-
-			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
-
-			go func() {
-				var stats runtime.MemStats
-
-				start := time.Now().UnixNano()
-
-				for {
-					runtime.ReadMemStats(&stats)
-					if context.fileMemStats != nil {
-						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
-							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
-						time.Sleep(interval)
-					} else {
-						break
-					}
-				}
-			}()
-		}
-	}
-
-	return nil
-}
 
 // ShutdownContext shuts context down
 func ShutdownContext() {
-	if aptly.EnableDebug {
+	context.Shutdown()
-		if context.fileMemProfile != nil {
+}
-			pprof.WriteHeapProfile(context.fileMemProfile)
+
-			context.fileMemProfile.Close()
+// CleanupContext does partial shutdown of context
-			context.fileMemProfile = nil
+func CleanupContext() {
-		}
+	context.Cleanup()
-		if context.fileCPUProfile != nil {
+}
-			pprof.StopCPUProfile()
+
-			context.fileCPUProfile.Close()
+// InitContext initializes context with default settings
-			context.fileCPUProfile = nil
+func InitContext(flags *flag.FlagSet) error {
-		}
+	var err error
-		if context.fileMemProfile != nil {
+
-			context.fileMemProfile.Close()
+	if context != nil {
-			context.fileMemProfile = nil
+		panic("context already initialized")
-		}
+	}
-	}
+
-	if context.database != nil {
+	context, err = ctx.NewContext(flags)
-		context.database.Close()
+
-	}
+	return err
-	if context.downloader != nil {
-		context.downloader.Shutdown()
-	}
-	if context.progress != nil {
-		context.progress.Shutdown()
-	}
 }

cmd/db.go

@@ -1,8 +1,7 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
+	"github.com/smira/commander"
-	"github.com/gonuts/flag"
 )
 
 func makeCmdDb() *commander.Command {
@@ -11,7 +10,7 @@ func makeCmdDb() *commander.Command {
 		Short:     "manage aptly's internal database and package pool",
 		Subcommands: []*commander.Command{
 			makeCmdDbCleanup(),
+			makeCmdDbRecover(),
 		},
-		Flag: *flag.NewFlagSet("aptly-db", flag.ExitOnError),
 	}
 }

cmd/db_cleanup.go

@@ -2,11 +2,11 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
 	"sort"
+	"strings"
 )
 
 // aptly db cleanup
@@ -15,21 +15,101 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
+	verbose := context.Flags().Lookup("verbose").Value.Get().(bool)
+	dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
+
 	// collect information about references packages...
-	existingPackageRefs := debian.NewPackageRefList()
+	existingPackageRefs := deb.NewPackageRefList()
 
-	context.progress.Printf("Loading mirrors, local repos and snapshots...\n")
+	// used only in verbose mode to report package use source
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
+	packageRefSources := map[string][]string{}
-	err = repoCollection.ForEach(func(repo *debian.RemoteRepo) error {
+
-		err := repoCollection.LoadComplete(repo)
+	context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|")
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
+	}
+	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
+		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
 		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("mirror %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
+	}
+	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
+		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+
+		if repo.RefList() != nil {
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("local repo %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
+	}
+	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
+		}
+
+		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+
+		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+
+		if verbose {
+			description := fmt.Sprintf("snapshot %s", snapshot.Name)
+			snapshot.RefList().ForEach(func(key []byte) error {
+				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+				return nil
+			})
 		}
 		return nil
 	})
@@ -37,28 +117,32 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	if verbose {
-	err = localRepoCollection.ForEach(func(repo *debian.LocalRepo) error {
+		context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
-		err := localRepoCollection.LoadComplete(repo)
-		if err != nil {
-			return err
-		}
-		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
-		}
-		return nil
-	})
-	if err != nil {
-		return err
 	}
-
+	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+		if verbose {
-	err = snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
+			context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
-		err := snapshotCollection.LoadComplete(snapshot)
+		}
+		if published.SourceKind != "local" {
+			return nil
+		}
+		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
 		if err != nil {
 			return err
 		}
-		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false)
+
+		for _, component := range published.Components() {
+			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+			if verbose {
+				description := fmt.Sprintf("published repository %s:%s/%s component %s",
+					published.Storage, published.Prefix, published.Distribution, component)
+				published.RefList(component).ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
+		}
 		return nil
 	})
 	if err != nil {
@@ -66,44 +150,72 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	}
 
 	// ... and compare it to the list of all packages
-	context.progress.Printf("Loading list of all packages...\n")
+	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
-	packageCollection := debian.NewPackageCollection(context.database)
+	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
-	allPackageRefs := packageCollection.AllPackageRefs()
 
 	toDelete := allPackageRefs.Substract(existingPackageRefs)
 
 	// delete packages that are no longer referenced
-	context.progress.Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())
 
-	context.database.StartBatch()
+	// database can't err as collection factory already constructed
-	err = toDelete.ForEach(func(ref []byte) error {
+	db, _ := context.Database()
-		return packageCollection.DeleteByKey(ref)
-	})
-	if err != nil {
-		return err
-	}
 
-	err = context.database.FinishBatch()
+	if toDelete.Len() > 0 {
-	if err != nil {
+		if verbose {
-		return fmt.Errorf("unable to write to DB: %s", err)
+			context.Progress().ColoredPrintf("@{r}List of package keys to delete:@|")
+			err = toDelete.ForEach(func(ref []byte) error {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", string(ref))
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if !dryRun {
+			db.StartBatch()
+			err = toDelete.ForEach(func(ref []byte) error {
+				return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
+			})
+			if err != nil {
+				return err
+			}
+
+			err = db.FinishBatch()
+			if err != nil {
+				return fmt.Errorf("unable to write to DB: %s", err)
+			}
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped deletion, as -dry-run has been requested.@|")
+		}
 	}
 
 	// now, build a list of files that should be present in Repository (package pool)
-	context.progress.Printf("Building list of files referenced by packages...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())
-	context.progress.InitBar(int64(existingPackageRefs.Len()), false)
+	context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
 
 	err = existingPackageRefs.ForEach(func(key []byte) error {
-		pkg, err := packageCollection.ByKey(key)
+		pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
-		if err != nil {
+		if err2 != nil {
-			return err
+			tail := ""
+			if verbose {
+				tail = fmt.Sprintf(" (sources: %s)", strings.Join(packageRefSources[string(key)], ", "))
+			}
+			if dryRun {
+				context.Progress().ColoredPrintf("@{r!}Unresolvable package reference, skipping (-dry-run): %s: %s%s",
+					string(key), err2, tail)
+				return nil
+			}
+			return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
 		}
-		paths, err := pkg.FilepathList(context.packagePool)
+		paths, err2 := pkg.FilepathList(context.PackagePool())
-		if err != nil {
+		if err2 != nil {
-			return err
+			return err2
 		}
 		referencedFiles = append(referencedFiles, paths...)
-		context.progress.AddBar(1)
+		context.Progress().AddBar(1)
 
 		return nil
 	})
@@ -112,11 +224,11 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	}
 
 	sort.Strings(referencedFiles)
-	context.progress.ShutdownBar()
+	context.Progress().ShutdownBar()
 
 	// build a list of files in the package pool
-	context.progress.Printf("Building list of files in package pool...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files in package pool...@|")
-	existingFiles, err := context.packagePool.FilepathList(context.progress)
+	existingFiles, err := context.PackagePool().FilepathList(context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to collect file paths: %s", err)
 	}
@@ -125,24 +237,44 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
 
 	// delete files that are no longer referenced
-	context.progress.Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced files (%d)...@|", len(filesToDelete))
 
 	if len(filesToDelete) > 0 {
-		context.progress.InitBar(int64(len(filesToDelete)), false)
+		if verbose {
-		totalSize := int64(0)
+			context.Progress().ColoredPrintf("@{r}List of files to be deleted:@|")
-		for _, file := range filesToDelete {
+			for _, file := range filesToDelete {
-			size, err := context.packagePool.Remove(file)
+				context.Progress().ColoredPrintf(" - @{r}%s@|", file)
-			if err != nil {
-				return err
 			}
-
-			context.progress.AddBar(1)
-			totalSize += size
 		}
-		context.progress.ShutdownBar()
 
-		context.progress.Printf("Disk space freed: %.2f GiB...\n", float64(totalSize)/1024.0/1024.0/1024.0)
+		if !dryRun {
+			context.Progress().InitBar(int64(len(filesToDelete)), false)
+
+			var size, totalSize int64
+			for _, file := range filesToDelete {
+				size, err = context.PackagePool().Remove(file)
+				if err != nil {
+					return err
+				}
+
+				context.Progress().AddBar(1)
+				totalSize += size
+			}
+			context.Progress().ShutdownBar()
+
+			context.Progress().ColoredPrintf("@{w!}Disk space freed: %s...@|", utils.HumanBytes(totalSize))
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped file deletion, as -dry-run has been requested.@|")
+		}
 	}
+
+	if !dryRun {
+		context.Progress().ColoredPrintf("@{w!}Compacting database...@|")
+		err = db.CompactDB()
+	} else {
+		context.Progress().ColoredPrintf("@{y!}Skipped DB compaction, as -dry-run has been requested.@|")
+	}
+
 	return err
 }
 
@@ -159,8 +291,10 @@ Example:
 
   $ aptly db cleanup
 `,
-		Flag: *flag.NewFlagSet("aptly-db-cleanup", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("verbose", false, "be verbose when loading objects/removing them")
+	cmd.Flag.Bool("dry-run", false, "don't delete anything")
+
 	return cmd
 }

cmd/db_recover.go

@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"github.com/smira/aptly/database"
+	"github.com/smira/commander"
+)
+
+// aptly db recover
+func aptlyDbRecover(cmd *commander.Command, args []string) error {
+	var err error
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	context.Progress().Printf("Recovering database...\n")
+	err = database.RecoverDB(context.DBPath())
+
+	return err
+}
+
+func makeCmdDbRecover() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyDbRecover,
+		UsageLine: "recover",
+		Short:     "recover DB after crash",
+		Long: `
+Database recover does its' best to recover the database after a crash.
+It is recommended to backup the DB before running recover.
+
+Example:
+
+  $ aptly db recover
+`,
+	}
+
+	return cmd
+}

cmd/graph.go

@@ -2,145 +2,28 @@ package cmd
 
 import (
 	"bytes"
-	"code.google.com/p/gographviz"
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
-	"github.com/smira/aptly/debian"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"strings"
 )
 
-func graphvizEscape(s string) string {
-	return fmt.Sprintf("\"%s\"", strings.Replace(s, "\"", "\\\"", 0))
-}
-
 func aptlyGraph(cmd *commander.Command, args []string) error {
 	var err error
 
-	graph := gographviz.NewGraph()
+	if len(args) != 0 {
-	graph.SetDir(true)
+		cmd.Usage()
-	graph.SetName("aptly")
+		return commander.ErrCommandError
-
-	existingNodes := map[string]bool{}
-
-	fmt.Printf("Loading mirrors...\n")
-
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-
-	err = repoCollection.ForEach(func(repo *debian.RemoteRepo) error {
-		err := repoCollection.LoadComplete(repo)
-		if err != nil {
-			return err
-		}
-
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "darkgoldenrod1",
-			"label": graphvizEscape(fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
-				repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "),
-				strings.Join(repo.Architectures, ", "), repo.NumPackages())),
-		})
-		existingNodes[repo.UUID] = true
-		return nil
-	})
-
-	if err != nil {
-		return err
 	}
 
-	fmt.Printf("Loading local repos...\n")
-
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
-
-	err = localRepoCollection.ForEach(func(repo *debian.LocalRepo) error {
-		err := localRepoCollection.LoadComplete(repo)
-		if err != nil {
-			return err
-		}
-
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "mediumseagreen",
-			"label": graphvizEscape(fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
-				repo.Name, repo.Comment, repo.NumPackages())),
-		})
-		existingNodes[repo.UUID] = true
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Loading snapshots...\n")
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-
-	snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
-		existingNodes[snapshot.UUID] = true
-		return nil
-	})
-
-	err = snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
-		err := snapshotCollection.LoadComplete(snapshot)
-		if err != nil {
-			return err
-		}
-
-		description := snapshot.Description
-		if snapshot.SourceKind == "repo" {
-			description = "Snapshot from repo"
-		}
-
-		graph.AddNode("aptly", graphvizEscape(snapshot.UUID), map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "cadetblue1",
-			"label":     graphvizEscape(fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages())),
-		})
-
-		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
-			for _, uuid := range snapshot.SourceIDs {
-				_, exists := existingNodes[uuid]
-				if exists {
-					graph.AddEdge(graphvizEscape(uuid), "", graphvizEscape(snapshot.UUID), "", true, nil)
-				}
-			}
-		}
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Loading published repos...\n")
-
-	publishedCollection := debian.NewPublishedRepoCollection(context.database)
-
-	publishedCollection.ForEach(func(repo *debian.PublishedRepo) error {
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "darkolivegreen1",
-			"label":     graphvizEscape(fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution, repo.Component, strings.Join(repo.Architectures, ", "))),
-		})
-
-		_, exists := existingNodes[repo.SnapshotUUID]
-		if exists {
-			graph.AddEdge(graphvizEscape(repo.SnapshotUUID), "", graphvizEscape(repo.UUID), "", true, nil)
-		}
-
-		return nil
-	})
-
 	fmt.Printf("Generating graph...\n")
+	graph, err := deb.BuildGraph(context.CollectionFactory())
+	if err != nil {
+		return err
+	}
 
 	buf := bytes.NewBufferString(graph.String())
 
@@ -198,13 +81,12 @@ func makeCmdGraph() *commander.Command {
 		Long: `
 Command graph displays relationship between mirrors, local repositories,
 snapshots and published repositories using graphviz package to render
-graph as image.
+graph as an image.
 
 Example:
 
   $ aptly graph
 `,
-		Flag: *flag.NewFlagSet("aptly-graph", flag.ExitOnError),
 	}
 
 	return cmd

cmd/mirror.go

@@ -1,19 +1,21 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"strings"
 )
 
-func getVerifier(cmd *commander.Command) (utils.Verifier, error) {
+func getVerifier(flags *flag.FlagSet) (utils.Verifier, error) {
-	if utils.Config.GpgDisableVerify || cmd.Flag.Lookup("ignore-signatures").Value.Get().(bool) {
+	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
 		return nil, nil
 	}
 
+	keyRings := flags.Lookup("keyring").Value.Get().([]string)
+
 	verifier := &utils.GpgVerifier{}
-	for _, keyRing := range keyRings.keyRings {
+	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}
 
@@ -42,8 +44,6 @@ func (k *keyRingsFlag) String() string {
 	return strings.Join(k.keyRings, ",")
 }
 
-var keyRings = keyRingsFlag{}
-
 func makeCmdMirror() *commander.Command {
 	return &commander.Command{
 		UsageLine: "mirror",
@@ -54,7 +54,9 @@ func makeCmdMirror() *commander.Command {
 			makeCmdMirrorShow(),
 			makeCmdMirrorDrop(),
 			makeCmdMirrorUpdate(),
+			makeCmdMirrorRename(),
+			makeCmdMirrorEdit(),
+			makeCmdMirrorSearch(),
 		},
-		Flag: *flag.NewFlagSet("aptly-mirror", flag.ExitOnError),
 	}
 }

cmd/mirror_create.go

@@ -2,10 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/aptly/query"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
-	"github.com/smira/aptly/utils"
+	"github.com/smira/flag"
 	"strings"
 )
 
@@ -13,10 +13,11 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	var err error
 	if !(len(args) == 2 && strings.HasPrefix(args[1], "ppa:") || len(args) >= 3) {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	downloadSources := utils.Config.DownloadSourcePackages || cmd.Flag.Lookup("with-sources").Value.Get().(bool)
+	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
+	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -25,7 +26,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 
 	mirrorName = args[0]
 	if len(args) == 2 {
-		archiveURL, distribution, components, err = debian.ParsePPA(args[1])
+		archiveURL, distribution, components, err = deb.ParsePPA(args[1], context.Config())
 		if err != nil {
 			return err
 		}
@@ -33,24 +34,34 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		archiveURL, distribution, components = args[1], args[2], args[3:]
 	}
 
-	repo, err := debian.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.architecturesList, downloadSources)
+	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
+		downloadSources, downloadUdebs)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
-	verifier, err := getVerifier(cmd)
+	repo.Filter = context.Flags().Lookup("filter").Value.String()
+	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
+	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to create mirror: %s", err)
+		}
+	}
+
+	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.downloader, verifier)
+	err = repo.Fetch(context.Downloader(), verifier)
 	if err != nil {
 		return fmt.Errorf("unable to fetch mirror: %s", err)
 	}
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
+	err = context.CollectionFactory().RemoteRepoCollection().Add(repo)
-
-	err = repoCollection.Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add mirror: %s", err)
 	}
@@ -66,7 +77,8 @@ func makeCmdMirrorCreate() *commander.Command {
 		Short:     "create new mirror",
 		Long: `
 Creates mirror <name> of remote repository, aptly supports both regular and flat Debian repositories exported
-via HTTP. aptly would try download Release file from remote repository and verify its signature.
+via HTTP and FTP. aptly would try download Release file from remote repository and verify its' signature. Command
+line format resembles apt utlitily sources.list(5).
 
 PPA urls could specified in short format:
 
@@ -81,7 +93,11 @@ Example:
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
-	cmd.Flag.Var(&keyRings, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd
 }

cmd/mirror_drop.go

@@ -2,30 +2,32 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/flag"
-	"github.com/smira/aptly/debian"
 )
 
 func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
-	repo, err := repoCollection.ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	force := cmd.Flag.Lookup("force").Value.Get().(bool)
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshotCollection := debian.NewSnapshotCollection(context.database)
+		snapshots := context.CollectionFactory().SnapshotCollection().ByRemoteRepoSource(repo)
-		snapshots := snapshotCollection.ByRemoteRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Mirror `%s` was used to create following snapshots:\n", repo.Name)
@@ -37,7 +39,7 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = repoCollection.Drop(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
@@ -54,7 +56,7 @@ func makeCmdMirrorDrop() *commander.Command {
 		Short:     "delete mirror",
 		Long: `
 Drop deletes information about remote repository mirror <name>. Package data is not deleted
-(it could be still used by other mirrors or snapshots).  If mirror is used as source
+(since it could still be used by other mirrors or snapshots).  If mirror is used as source
 to create a snapshot, aptly would refuse to delete such mirror, use flag -force to override.
 
 Example:

cmd/mirror_edit.go

@@ -0,0 +1,91 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	context.Flags().Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "filter":
+			repo.Filter = flag.Value.String()
+		case "filter-with-deps":
+			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-sources":
+			repo.DownloadSources = flag.Value.Get().(bool)
+		case "with-udebs":
+			repo.DownloadUdebs = flag.Value.Get().(bool)
+		}
+	})
+
+	if repo.IsFlat() && repo.DownloadUdebs {
+		return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
+	}
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
+	if context.GlobalFlags().Lookup("architectures").Value.String() != "" {
+		repo.Architectures = context.ArchitecturesList()
+
+		err = repo.Fetch(context.Downloader(), nil)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fmt.Printf("Mirror %s successfully updated.\n", repo)
+	return err
+}
+
+func makeCmdMirrorEdit() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorEdit,
+		UsageLine: "edit <name>",
+		Short:     "edit mirror settings",
+		Long: `
+Command edit allows one to change settings of mirror:
+filters, list of architectures.
+
+Example:
+
+  $ aptly mirror edit -filter=nginx -filter-with-deps some-mirror
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-edit", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+
+	return cmd
+}

cmd/mirror_list.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
-	"github.com/smira/aptly/debian"
 	"sort"
 )
 
@@ -12,29 +11,42 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	if repoCollection.Len() > 0 {
+	repos := make([]string, context.CollectionFactory().RemoteRepoCollection().Len())
-		fmt.Printf("List of mirrors:\n")
+	i := 0
-		repos := make([]string, repoCollection.Len())
+	context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		i := 0
+		if raw {
-		repoCollection.ForEach(func(repo *debian.RemoteRepo) error {
+			repos[i] = repo.Name
+		} else {
 			repos[i] = repo.String()
-			i++
-			return nil
-		})
-
-		sort.Strings(repos)
-		for _, repo := range repos {
-			fmt.Printf(" * %s\n", repo)
 		}
+		i++
+		return nil
+	})
 
-		fmt.Printf("\nTo get more information about mirror, run `aptly mirror show <name>`.\n")
+	context.CloseDatabase()
+
+	sort.Strings(repos)
+
+	if raw {
+		for _, repo := range repos {
+			fmt.Printf("%s\n", repo)
+		}
 	} else {
-		fmt.Printf("No mirrors found, create one with `aptly mirror create ...`.\n")
+		if len(repos) > 0 {
+			fmt.Printf("List of mirrors:\n")
+			for _, repo := range repos {
+				fmt.Printf(" * %s\n", repo)
+			}
+
+			fmt.Printf("\nTo get more information about mirror, run `aptly mirror show <name>`.\n")
+		} else {
+			fmt.Printf("No mirrors found, create one with `aptly mirror create ...`.\n")
+		}
 	}
 	return err
 }
@@ -51,8 +63,9 @@ Example:
 
   $ aptly mirror list
 `,
-		Flag: *flag.NewFlagSet("aptly-mirror-list", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+
 	return cmd
 }

cmd/mirror_rename.go

@@ -0,0 +1,64 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyMirrorRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.RemoteRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().RemoteRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: mirror %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nMirror %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdMirrorRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames mirror",
+		Long: `
+Command changes name of the mirror.Mirror name should be unique.
+
+Example:
+
+  $ aptly mirror rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/mirror_search.go

@@ -0,0 +1,26 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdMirrorSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> <package-query>",
+		Short:     "search mirror for packages matching query",
+		Long: `
+Command search displays list of packages in mirror that match package query
+
+Example:
+
+    $ aptly mirror search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+
+	return cmd
+}

cmd/mirror_show.go

@@ -2,10 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"strings"
 )
 
@@ -13,23 +13,25 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
-	repo, err := repoCollection.ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = repoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
 	fmt.Printf("Name: %s\n", repo.Name)
+	if repo.Status == deb.MirrorUpdating {
+		fmt.Printf("Status: In Update (PID %d)\n", repo.WorkerPID)
+	}
 	fmt.Printf("Archive Root URL: %s\n", repo.ArchiveRoot)
 	fmt.Printf("Distribution: %s\n", repo.Distribution)
 	fmt.Printf("Components: %s\n", strings.Join(repo.Components, ", "))
@@ -39,6 +41,19 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		downloadSources = "yes"
 	}
 	fmt.Printf("Download Sources: %s\n", downloadSources)
+	downloadUdebs := "no"
+	if repo.DownloadUdebs {
+		downloadUdebs = "yes"
+	}
+	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
+	if repo.Filter != "" {
+		fmt.Printf("Filter: %s\n", repo.Filter)
+		filterWithDeps := "no"
+		if repo.FilterWithDeps {
+			filterWithDeps = "yes"
+		}
+		fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
+	}
 	if repo.LastDownloadDate.IsZero() {
 		fmt.Printf("Last update: never\n")
 	} else {
@@ -51,7 +66,7 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		fmt.Printf("%s: %s\n", k, repo.Meta[k])
 	}
 
-	withPackages := cmd.Flag.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		if repo.LastDownloadDate.IsZero() {
 			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
@@ -69,7 +84,7 @@ func makeCmdMirrorShow() *commander.Command {
 		UsageLine: "show <name>",
 		Short:     "show details about mirror",
 		Long: `
-Shows detailed information about mirror.
+Shows detailed information about the mirror.
 
 Example:
 

cmd/mirror_update.go

@@ -2,56 +2,167 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/aptly/query"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"os"
+	"os/signal"
+	"strings"
 )
 
 func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
+	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
-	repo, err := repoCollection.ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = repoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	ignoreMismatch := cmd.Flag.Lookup("ignore-checksums").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
+	if !force {
+		err = repo.CheckLock()
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
 
-	verifier, err := getVerifier(cmd)
+	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
+
+	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.downloader, verifier)
+	err = repo.Fetch(context.Downloader(), verifier)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	packageCollection := debian.NewPackageCollection(context.database)
+	context.Progress().Printf("Downloading & parsing package files...\n")
-
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch)
-	err = repo.Download(context.progress, context.downloader, packageCollection, context.packagePool, ignoreMismatch)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = repoCollection.Update(repo)
+	if repo.Filter != "" {
+		context.Progress().Printf("Applying filter...\n")
+		var filterQuery deb.PackageQuery
+
+		filterQuery, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+
+		var oldLen, newLen int
+		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+		context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen)
+	}
+
+	var (
+		downloadSize int64
+		queue        []deb.PackageDownloadTask
+	)
+
+	context.Progress().Printf("Building download queue...\n")
+	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool())
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	context.progress.Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
+	defer func() {
+		// on any interruption, unlock the mirror
+		err := context.ReOpenDatabase()
+		if err == nil {
+			repo.MarkAsIdle()
+			context.CollectionFactory().RemoteRepoCollection().Update(repo)
+		}
+	}()
+
+	repo.MarkAsUpdating()
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = context.CloseDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	// Catch ^C
+	sigch := make(chan os.Signal)
+	signal.Notify(sigch, os.Interrupt)
+
+	count := len(queue)
+	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
+
+	// Download from the queue
+	context.Progress().InitBar(downloadSize, true)
+
+	// Download all package files
+	ch := make(chan error, count)
+
+	// In separate goroutine (to avoid blocking main), push queue to downloader
+	go func() {
+		for _, task := range queue {
+			context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
+		}
+
+		// We don't need queue after this point
+		queue = nil
+	}()
+
+	// Wait for all downloads to finish
+	errors := make([]string, 0)
+
+	for count > 0 {
+		select {
+		case <-sigch:
+			signal.Stop(sigch)
+			return fmt.Errorf("unable to update: interrupted")
+		case err = <-ch:
+			if err != nil {
+				errors = append(errors, err.Error())
+			}
+			count--
+		}
+	}
+
+	context.Progress().ShutdownBar()
+	signal.Stop(sigch)
+
+	if len(errors) > 0 {
+		return fmt.Errorf("unable to update: download errors:\n  %s\n", strings.Join(errors, "\n  "))
+	}
+
+	err = context.ReOpenDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	repo.FinalizeDownload()
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
 	return err
 }
 
@@ -62,8 +173,8 @@ func makeCmdMirrorUpdate() *commander.Command {
 		Short:     "update mirror",
 		Long: `
 Updates remote mirror (downloads package files and meta information). When mirror is created,
-this command should be run for the first time to fetch mirror contents. This command could be
+this command should be run for the first time to fetch mirror contents. This command can be
-run many times to get updated repository contents. If interrupted, command could be restarted safely.
+run multiple times to get updated repository contents. If interrupted, command can be safely restarted.
 
 Example:
 
@@ -72,9 +183,11 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-update", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("force", false, "force update mirror even if it is locked by another process")
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
-	cmd.Flag.Var(&keyRings, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd
 }

cmd/package.go

@@ -0,0 +1,16 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdPackage() *commander.Command {
+	return &commander.Command{
+		UsageLine: "package",
+		Short:     "operations on packages",
+		Subcommands: []*commander.Command{
+			makeCmdPackageSearch(),
+			makeCmdPackageShow(),
+		},
+	}
+}

cmd/package_search.go

@@ -0,0 +1,52 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPackageSearch(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	q, err := query.Parse(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	result := q.Query(context.CollectionFactory().PackageCollection())
+	if result.Len() == 0 {
+		return fmt.Errorf("no results")
+	}
+
+	result.ForEach(func(p *deb.Package) error {
+		context.Progress().Printf("%s\n", p)
+		return nil
+	})
+
+	return err
+}
+
+func makeCmdPackageSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageSearch,
+		UsageLine: "search <package-query>",
+		Short:     "search for packages matching query",
+		Long: `
+Command search displays list of packages in whole DB that match package query
+
+Example:
+
+    $ aptly package search '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-search", flag.ExitOnError),
+	}
+
+	return cmd
+}

cmd/package_show.go

@@ -0,0 +1,137 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"os"
+)
+
+func printReferencesTo(p *deb.Package) (err error) {
+	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  mirror %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  local repo %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+		if snapshot.RefList().Has(p) {
+			fmt.Printf("  snapshot %s\n", snapshot)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func aptlyPackageShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	q, err := query.Parse(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	withFiles := context.Flags().Lookup("with-files").Value.Get().(bool)
+	withReferences := context.Flags().Lookup("with-references").Value.Get().(bool)
+
+	w := bufio.NewWriter(os.Stdout)
+
+	result := q.Query(context.CollectionFactory().PackageCollection())
+
+	err = result.ForEach(func(p *deb.Package) error {
+		p.Stanza().WriteTo(w, p.IsSource, false)
+		w.Flush()
+		fmt.Printf("\n")
+
+		if withFiles {
+			fmt.Printf("Files in the pool:\n")
+			for _, f := range p.Files() {
+				path, err := context.PackagePool().Path(f.Filename, f.Checksums.MD5)
+				if err != nil {
+					return err
+				}
+				fmt.Printf("  %s\n", path)
+			}
+			fmt.Printf("\n")
+		}
+
+		if withReferences {
+			fmt.Printf("References to package:\n")
+			printReferencesTo(p)
+			fmt.Printf("\n")
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdPackageShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageShow,
+		UsageLine: "show <package-query>",
+		Short:     "show details about packages matching query",
+		Long: `
+Command shows displays detailed meta-information about packages
+matching query. Information from Debian control file is displayed.
+Optionally information about package files and
+inclusion into mirrors/snapshots/local repos is shown.
+
+Example:
+
+    $ aptly package show nginx-light_1.2.1-2.2+wheezy2_i386'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-files", false, "display information about files from package pool")
+	cmd.Flag.Bool("with-references", false, "display information about mirrors, snapshots and local repos referencing this package")
+
+	return cmd
+}

cmd/publish.go

@@ -1,19 +1,21 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
-func getSigner(cmd *commander.Command) (utils.Signer, error) {
+func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
-	if cmd.Flag.Lookup("skip-signing").Value.Get().(bool) || utils.Config.GpgDisableSign {
+	if LookupOption(context.Config().GpgDisableSign, flags, "skip-signing") {
 		return nil, nil
 	}
 
 	signer := &utils.GpgSigner{}
-	signer.SetKey(cmd.Flag.Lookup("gpg-key").Value.String())
+	signer.SetKey(flags.Lookup("gpg-key").Value.String())
-	signer.SetKeyRing(cmd.Flag.Lookup("keyring").Value.String(), cmd.Flag.Lookup("secret-keyring").Value.String())
+	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
+	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
+	signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
 
 	err := signer.Init()
 	if err != nil {
@@ -29,10 +31,12 @@ func makeCmdPublish() *commander.Command {
 		UsageLine: "publish",
 		Short:     "manage published repositories",
 		Subcommands: []*commander.Command{
-			makeCmdPublishSnapshot(),
-			makeCmdPublishList(),
 			makeCmdPublishDrop(),
+			makeCmdPublishList(),
+			makeCmdPublishRepo(),
+			makeCmdPublishSnapshot(),
+			makeCmdPublishSwitch(),
+			makeCmdPublishUpdate(),
 		},
-		Flag: *flag.NewFlagSet("aptly-publish", flag.ExitOnError),
 	}
 }

cmd/publish_drop.go

@@ -2,33 +2,33 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
-	"github.com/smira/aptly/debian"
 )
 
 func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 1 || len(args) > 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	if len(args) == 2 {
-		prefix = args[1]
+		param = args[1]
 	}
 
-	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
+	storage, prefix := deb.ParsePrefix(param)
 
-	err = publishedCollecton.Remove(context.publishedStorage, prefix, distribution)
+	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
+		context.CollectionFactory(), context.Progress(), context.Flags().Lookup("force-drop").Value.Get().(bool))
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	fmt.Printf("\nPublished repositroy has been removed successfully.\n")
+	context.Progress().Printf("\nPublished repository has been removed successfully.\n")
 
 	return err
 }
@@ -36,18 +36,19 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 func makeCmdPublishDrop() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishDrop,
-		UsageLine: "drop <distribution> [<prefix>]",
+		UsageLine: "drop <distribution> [[<endpoint>:]<prefix>]",
 		Short:     "remove published repository",
 		Long: `
-Command removes whatever has been published under specified <prefix> and
+Command removes whatever has been published under specified <prefix>,
-<distribution> name.
+publishing <endpoint> and <distribution> name.
 
 Example:
 
     $ aptly publish drop wheezy
 `,
-		Flag: *flag.NewFlagSet("aptly-publish-drop", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
+
 	return cmd
 }

cmd/publish_list.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
-	"github.com/smira/aptly/debian"
 	"sort"
 )
 
@@ -12,26 +11,24 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
 
-	if publishedCollecton.Len() == 0 {
+	published := make([]string, 0, context.CollectionFactory().PublishedRepoCollection().Len())
-		fmt.Printf("No snapshots have been published. Publish a snapshot by running `aptly publish snapshot ...`.\n")
-		return err
-	}
 
-	published := make([]string, 0, publishedCollecton.Len())
+	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-
+		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
-	err = publishedCollecton.ForEach(func(repo *debian.PublishedRepo) error {
-		err := publishedCollecton.LoadComplete(repo, snapshotCollection)
 		if err != nil {
 			return err
 		}
 
-		published = append(published, repo.String())
+		if raw {
+			published = append(published, fmt.Sprintf("%s %s", repo.StoragePrefix(), repo.Distribution))
+		} else {
+			published = append(published, repo.String())
+		}
 		return nil
 	})
 
@@ -39,12 +36,25 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load list of repos: %s", err)
 	}
 
+	context.CloseDatabase()
+
 	sort.Strings(published)
 
-	fmt.Printf("Published repositories:\n")
+	if raw {
+		for _, info := range published {
+			fmt.Printf("%s\n", info)
+		}
+	} else {
+		if len(published) == 0 {
+			fmt.Printf("No snapshots/local repos have been published. Publish a snapshot by running `aptly publish snapshot ...`.\n")
+			return err
+		}
 
-	for _, description := range published {
+		fmt.Printf("Published repositories:\n")
-		fmt.Printf("  * %s\n", description)
+
+		for _, description := range published {
+			fmt.Printf("  * %s\n", description)
+		}
 	}
 
 	return err
@@ -62,8 +72,9 @@ Example:
 
     $ aptly publish list
 `,
-		Flag: *flag.NewFlagSet("aptly-publish-list", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+
 	return cmd
 }

cmd/publish_repo.go

@@ -0,0 +1,49 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdPublishRepo() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSnapshotOrRepo,
+		UsageLine: "repo <name> [[<endpoint>:]<prefix>]",
+		Short:     "publish local repository",
+		Long: `
+Command publishes current state of local repository ready to be consumed
+by apt tools. Published repostiories appear under rootDir/public directory.
+Valid GPG key is required for publishing.
+
+Multiple component repository could be published by specifying several
+components split by commas via -component flag and multiple local
+repositories as the arguments:
+
+    aptly publish repo -component=main,contrib repo-main repo-contrib
+
+It is not recommended to publish local repositories directly unless the
+repository is for testing purposes and changes happen frequently. For
+production usage please take snapshot of repository and publish it
+using publish snapshot command.
+
+Example:
+
+    $ aptly publish repo testing
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-repo", flag.ExitOnError),
+	}
+	cmd.Flag.String("distribution", "", "distribution name to publish")
+	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+
+	return cmd
+}

cmd/publish_snapshot.go

@@ -2,124 +2,192 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/aptly"
-	"github.com/gonuts/flag"
+	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/debian"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"strings"
 )
 
-func aptlyPublishSnapshot(cmd *commander.Command, args []string) error {
+func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	var err error
-	if len(args) < 1 || len(args) > 2 {
+
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(args) < len(components) || len(args) > len(components)+1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	name := args[0]
+	var param string
-
+	if len(args) == len(components)+1 {
-	var prefix string
+		param = args[len(components)]
-	if len(args) == 2 {
+		args = args[0 : len(args)-1]
-		prefix = args[1]
 	} else {
-		prefix = ""
+		param = ""
 	}
+	storage, prefix := deb.ParsePrefix(param)
 
-	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
+	var (
+		sources = []interface{}{}
+		message string
+	)
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	if cmd.Name() == "snapshot" {
-	snapshot, err := snapshotCollection.ByName(name)
+		var (
-	if err != nil {
+			snapshot     *deb.Snapshot
-		return fmt.Errorf("unable to publish: %s", err)
+			emptyWarning = false
-	}
+			parts        = []string{}
+		)
 
-	err = snapshotCollection.LoadComplete(snapshot)
+		for _, name := range args {
-	if err != nil {
+			snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
-		return fmt.Errorf("unable to publish: %s", err)
+			if err != nil {
-	}
+				return fmt.Errorf("unable to publish: %s", err)
+			}
 
-	var sourceRepo *debian.RemoteRepo
+			err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
 
-	if snapshot.SourceKind == "repo" && len(snapshot.SourceIDs) == 1 {
+			sources = append(sources, snapshot)
-		repoCollection := debian.NewRemoteRepoCollection(context.database)
+			parts = append(parts, snapshot.Name)
 
-		sourceRepo, _ = repoCollection.ByUUID(snapshot.SourceIDs[0])
+			if snapshot.NumPackages() == 0 {
-	}
+				emptyWarning = true
+			}
+		}
 
-	component := cmd.Flag.Lookup("component").Value.String()
+		if len(parts) == 1 {
-	if component == "" {
+			message = fmt.Sprintf("Snapshot %s has", parts[0])
-		if sourceRepo != nil && len(sourceRepo.Components) == 1 {
-			component = sourceRepo.Components[0]
 		} else {
-			component = "main"
+			message = fmt.Sprintf("Snapshots %s have", strings.Join(parts, ", "))
+
 		}
+
+		if emptyWarning {
+			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
+		}
+	} else if cmd.Name() == "repo" {
+		var (
+			localRepo    *deb.LocalRepo
+			emptyWarning = false
+			parts        = []string{}
+		)
+
+		for _, name := range args {
+			localRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			err = context.CollectionFactory().LocalRepoCollection().LoadComplete(localRepo)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			sources = append(sources, localRepo)
+			parts = append(parts, localRepo.Name)
+
+			if localRepo.NumPackages() == 0 {
+				emptyWarning = true
+			}
+		}
+
+		if len(parts) == 1 {
+			message = fmt.Sprintf("Local repo %s has", parts[0])
+		} else {
+			message = fmt.Sprintf("Local repos %s have", strings.Join(parts, ", "))
+
+		}
+
+		if emptyWarning {
+			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
+		}
+	} else {
+		panic("unknown command")
 	}
 
-	distribution := cmd.Flag.Lookup("distribution").Value.String()
+	distribution := context.Flags().Lookup("distribution").Value.String()
-	if distribution == "" {
-		if sourceRepo != nil {
-			distribution = sourceRepo.Distribution
-		}
 
-		if distribution == "" {
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
-			return fmt.Errorf("unable to guess distribution name, please specify explicitly")
-		}
-	}
-
-	published, err := debian.NewPublishedRepo(prefix, distribution, component, context.architecturesList, snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
+	published.Origin = cmd.Flag.Lookup("origin").Value.String()
+	published.Label = cmd.Flag.Lookup("label").Value.String()
 
-	duplicate := publishedCollecton.CheckDuplicate(published)
+	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
-		publishedCollecton.LoadComplete(duplicate, snapshotCollection)
+		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
 		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 	}
 
-	signer, err := getSigner(cmd)
+	signer, err := getSigner(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	packageCollection := debian.NewPackageCollection(context.database)
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
-	err = published.Publish(context.packagePool, context.publishedStorage, packageCollection, signer, context.progress)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = publishedCollecton.Add(published)
+	err = context.CollectionFactory().PublishedRepoCollection().Add(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	if prefix != "" && !strings.HasSuffix(prefix, "/") {
+	var repoComponents string
+	prefix, repoComponents, distribution = published.Prefix, strings.Join(published.Components(), " "), published.Distribution
+	if prefix == "." {
+		prefix = ""
+	} else if !strings.HasSuffix(prefix, "/") {
 		prefix += "/"
 	}
 
-	context.progress.Printf("\nSnapshot %s has been successfully published.\nPlease setup your webserver to serve directory '%s' with autoindexing.\n",
+	context.Progress().Printf("\n%s been successfully published.\n", message)
-		snapshot.Name, context.publishedStorage.PublicPath())
+
-	context.progress.Printf("Now you can add following line to apt sources:\n")
+	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.LocalPublishedStorage); ok {
-	context.progress.Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, component)
+		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
-	if utils.StrSliceHasItem(published.Architectures, "source") {
+			localStorage.PublicPath())
-		context.progress.Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, component)
 	}
-	context.progress.Printf("Don't forget to add your GPG key to apt with apt-key.\n")
+
-	context.progress.Printf("\nYou can also use `aptly serve` to publish your repositories over HTTP quickly.\n")
+	context.Progress().Printf("Now you can add following line to apt sources:\n")
+	context.Progress().Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
+	if utils.StrSliceHasItem(published.Architectures, "source") {
+		context.Progress().Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
+	}
+	context.Progress().Printf("Don't forget to add your GPG key to apt with apt-key.\n")
+	context.Progress().Printf("\nYou can also use `aptly serve` to publish your repositories over HTTP quickly.\n")
 
 	return err
 }
 
 func makeCmdPublishSnapshot() *commander.Command {
 	cmd := &commander.Command{
-		Run:       aptlyPublishSnapshot,
+		Run:       aptlyPublishSnapshotOrRepo,
-		UsageLine: "snapshot <name> [<prefix>]",
+		UsageLine: "snapshot <name> [[<endpoint>:]<prefix>]",
 		Short:     "publish snapshot",
 		Long: `
-Command publish publishes snapshot as Debian repository ready to be consumed
+Command publishes snapshot as Debian repository ready to be consumed
 by apt tools. Published repostiories appear under rootDir/public directory.
 Valid GPG key is required for publishing.
 
+Multiple component repository could be published by specifying several
+components split by commas via -component flag and multiple snapshots
+as the arguments:
+
+    aptly publish snapshot -component=main,contrib snap-main snap-contrib
+
 Example:
 
     $ aptly publish snapshot wheezy-main
@@ -127,11 +195,17 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-publish-snapshot", flag.ExitOnError),
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
-	cmd.Flag.String("component", "", "component name to publish")
+	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
-	cmd.Flag.String("keyring", "", "GPG keyring to use (instead of default)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd
 }

cmd/publish_switch.go

@@ -0,0 +1,150 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"strings"
+)
+
+func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
+	var err error
+
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(args) < len(components)+1 || len(args) > len(components)+2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	param := "."
+
+	var (
+		names    []string
+		snapshot *deb.Snapshot
+	)
+
+	if len(args) == len(components)+2 {
+		param = args[1]
+		names = args[2:]
+	} else {
+		names = args[1:]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	var published *deb.PublishedRepo
+
+	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	if published.SourceKind != "snapshot" {
+		return fmt.Errorf("unable to update: not a snapshot publish")
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	publishedComponents := published.Components()
+	if len(components) == 1 && len(publishedComponents) == 1 && components[0] == "" {
+		components = publishedComponents
+	}
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names))
+	}
+
+	for i, component := range components {
+		if !utils.StrSliceHasItem(publishedComponents, component) {
+			return fmt.Errorf("unable to switch: component %s is not in published repository", component)
+		}
+
+		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(names[i])
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
+
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
+
+		published.UpdateSnapshot(component, snapshot)
+	}
+
+	signer, err := getSigner(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG signer: %s", err)
+	}
+
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
+
+	return err
+}
+
+func makeCmdPublishSwitch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSwitch,
+		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
+		Short:     "update published repository by switching to new snapshot",
+		Long: `
+Command switches in-place published snapshots with new snapshot contents. All
+publishing parameters are preserved (architecture list, distribution,
+component).
+
+For multiple component repositories, flag -component should be given with
+list of components to update. Corresponding snapshots should be given in the
+same order, e.g.:
+
+	aptly publish switch -component=main,contrib wheezy wh-main wh-contrib
+
+Example:
+
+    $ aptly publish switch wheezy ppa wheezy-7.5
+
+This command would switch published repository (with one component) named ppa/wheezy
+(prefix ppa, dsitribution wheezy to new snapshot wheezy-7.5).
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
+	}
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+
+	return cmd
+}

cmd/publish_update.go

@@ -0,0 +1,108 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+	storage, prefix := deb.ParsePrefix(param)
+
+	var published *deb.PublishedRepo
+
+	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	if published.SourceKind != "local" {
+		return fmt.Errorf("unable to update: not a local repository publish")
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	components := published.Components()
+	for _, component := range components {
+		published.UpdateLocalRepo(component)
+	}
+
+	signer, err := getSigner(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG signer: %s", err)
+	}
+
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
+
+	return err
+}
+
+func makeCmdPublishUpdate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishUpdate,
+		UsageLine: "update <distribution> [[<endpoint>:]<prefix>]",
+		Short:     "update published local repository",
+		Long: `
+Command re-publishes (updates) published local repository. <distribution>
+and <prefix> should be occupied with local repository published
+using command aptly publish repo. Update happens in-place with
+minimum possible downtime for published repository.
+
+For multiple component published repositories, all local repositories
+are updated.
+
+Example:
+
+    $ aptly publish update wheezy ppa
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError),
+	}
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+
+	return cmd
+}

cmd/repo.go

@@ -1,8 +1,7 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
+	"github.com/smira/commander"
-	"github.com/gonuts/flag"
 )
 
 func makeCmdRepo() *commander.Command {
@@ -14,12 +13,14 @@ func makeCmdRepo() *commander.Command {
 			makeCmdRepoCopy(),
 			makeCmdRepoCreate(),
 			makeCmdRepoDrop(),
+			makeCmdRepoEdit(),
 			makeCmdRepoImport(),
 			makeCmdRepoList(),
 			makeCmdRepoMove(),
 			makeCmdRepoRemove(),
 			makeCmdRepoShow(),
+			makeCmdRepoRename(),
+			makeCmdRepoSearch(),
 		},
-		Flag: *flag.NewFlagSet("aptly-repo", flag.ExitOnError),
 	}
 }

cmd/repo_add.go

@@ -2,172 +2,68 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/aptly"
-	"github.com/gonuts/flag"
+	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/debian"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"os"
-	"path/filepath"
-	"sort"
-	"strings"
 )
 
 func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
 	verifier := &utils.GpgVerifier{}
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
-	repo, err := localRepoCollection.ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
-	err = localRepoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
-	context.progress.Printf("Loading packages...\n")
+	context.Progress().Printf("Loading packages...\n")
 
-	packageCollection := debian.NewPackageCollection(context.database)
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
-	list, err := debian.NewPackageListFromRefList(repo.RefList(), packageCollection, context.progress)
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	packageFiles := []string{}
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 
-	for _, location := range args[1:] {
+	var packageFiles, failedFiles []string
-		info, err := os.Stat(location)
-		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to process %s: %s@|", location, err)
-			continue
-		}
-		if info.IsDir() {
-			err = filepath.Walk(location, func(path string, info os.FileInfo, err error) error {
-				if err != nil {
-					return err
-				}
-				if info.IsDir() {
-					return nil
-				}
 
-				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
+	packageFiles, failedFiles, err = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
-					packageFiles = append(packageFiles, path)
+	if err != nil {
-				}
+		return fmt.Errorf("unable to collect package files: %s", err)
-
-				return nil
-			})
-		} else {
-			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
-				packageFiles = append(packageFiles, location)
-			} else {
-				context.progress.ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
-				continue
-			}
-		}
 	}
 
-	processedFiles := []string{}
+	var processedFiles, failedFiles2 []string
-	sort.Strings(packageFiles)
 
-	for _, file := range packageFiles {
+	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		var (
+		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()})
-			stanza debian.Stanza
+	failedFiles = append(failedFiles, failedFiles2...)
-			err    error
+	if err != nil {
-			p      *debian.Package
+		return fmt.Errorf("unable to import package files: %s", err)
-		)
-
-		candidateProcessedFiles := []string{}
-		isSourcePackage := strings.HasSuffix(file, ".dsc")
-
-		if isSourcePackage {
-			stanza, err = debian.GetControlFileFromDsc(file, verifier)
-
-			if err == nil {
-				stanza["Package"] = stanza["Source"]
-				delete(stanza, "Source")
-
-				p, err = debian.NewSourcePackageFromControlFile(stanza)
-			}
-		} else {
-			stanza, err = debian.GetControlFileFromDeb(file)
-			p = debian.NewPackageFromControlFile(stanza)
-		}
-		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
-			continue
-		}
-
-		checksums, err := utils.ChecksumsForFile(file)
-		if err != nil {
-			return err
-		}
-
-		if isSourcePackage {
-			p.UpdateFiles(append(p.Files(), debian.PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
-		} else {
-			p.UpdateFiles([]debian.PackageFile{debian.PackageFile{Filename: filepath.Base(file), Checksums: checksums}})
-		}
-
-		err = context.packagePool.Import(file, checksums.MD5)
-		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", file, err)
-			continue
-		}
-
-		candidateProcessedFiles = append(candidateProcessedFiles, file)
-
-		// go over all files, except for the last one (.dsc/.deb itself)
-		for _, f := range p.Files() {
-			if filepath.Base(f.Filename) == filepath.Base(file) {
-				continue
-			}
-			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
-			err = context.packagePool.Import(sourceFile, f.Checksums.MD5)
-			if err != nil {
-				context.progress.ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", sourceFile, err)
-				break
-			}
-
-			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
-		}
-		if err != nil {
-			// some files haven't been imported
-			continue
-		}
-
-		err = packageCollection.Update(p)
-		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to save package %s: %s@|", p, err)
-			continue
-		}
-
-		err = list.Add(p)
-		if err != nil {
-			context.progress.ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
-			continue
-		}
-
-		context.progress.ColoredPrintf("@g[+]@| %s added@|", p)
-		processedFiles = append(processedFiles, candidateProcessedFiles...)
 	}
 
-	repo.UpdateRefList(debian.NewPackageRefListFromPackageList(list))
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-	err = localRepoCollection.Update(repo)
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to save: %s", err)
 	}
 
-	if cmd.Flag.Lookup("remove-files").Value.Get().(bool) {
+	if context.Flags().Lookup("remove-files").Value.Get().(bool) {
 		processedFiles = utils.StrSliceDeduplicate(processedFiles)
 
 		for _, file := range processedFiles {
@@ -178,6 +74,15 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		}
 	}
 
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
 	return err
 }
 
@@ -187,11 +92,11 @@ func makeCmdRepoAdd() *commander.Command {
 		UsageLine: "add <name> <package file.deb>|<directory> ...",
 		Short:     "add packages to local repository",
 		Long: `
-Command adds packages to local repository from .deb (binary packages) and .dsc (source packages) files.
+Command adds packages to local repository from .deb, .udeb (binary packages) and .dsc (source packages) files.
-When importing from directory aptly would do recursive scan looking for all files matching *.deb or *.dsc
+When importing from directory aptly would do recursive scan looking for all files matching *.[u]deb or *.dsc
-patterns. Every file discovered would be analyzed to extract metadata, package would be created and added
+patterns. Every file discovered would be analyzed to extract metadata, package would then be created and added
-to database. Files would be imported to internal package pool. For source packages, all required files are
+to the database. Files would be imported to internal package pool. For source packages, all required files are
-added as well automatically. Extra files for source package should be in the same directory as *.dsc file.
+added automatically as well. Extra files for source package should be in the same directory as *.dsc file.
 
 Example:
 
@@ -201,6 +106,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("remove-files", false, "remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
 
 	return cmd
 }

cmd/repo_copy.go

@@ -1,17 +1,17 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
+	"github.com/smira/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/flag"
 )
 
 func makeCmdRepoCopy() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "copy <src-name> <dst-name> <package-spec> ...",
+		UsageLine: "copy <src-name> <dst-name> <package-query> ...",
 		Short:     "copy packages between local repositories",
 		Long: `
-Command copy copies packages matching <package-spec> from local repo
+Command copy copies packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
 Example:

cmd/repo_create.go

@@ -2,23 +2,23 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/flag"
 )
 
 func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	repo := debian.NewLocalRepo(args[0], cmd.Flag.Lookup("comment").Value.String())
+	repo := deb.NewLocalRepo(args[0], context.Flags().Lookup("comment").Value.String())
+	repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
+	repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
-
-	err = localRepoCollection.Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add local repo: %s", err)
 	}
@@ -45,6 +45,8 @@ Example:
 	}
 
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
+	cmd.Flag.String("distribution", "", "default distribution when publishing")
+	cmd.Flag.String("component", "main", "default component when publishing")
 
 	return cmd
 }

cmd/repo_drop.go

@@ -2,30 +2,41 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/flag"
-	"github.com/smira/aptly/debian"
 )
 
 func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
-	repo, err := localRepoCollection.ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	force := cmd.Flag.Lookup("force").Value.Get().(bool)
+	published := context.CollectionFactory().PublishedRepoCollection().ByLocalRepo(repo)
+	if len(published) > 0 {
+		fmt.Printf("Local repo `%s` is published currently:\n", repo.Name)
+		for _, repo := range published {
+			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+			if err != nil {
+				return fmt.Errorf("unable to load published: %s", err)
+			}
+			fmt.Printf(" * %s\n", repo)
+		}
+
+		return fmt.Errorf("unable to drop: local repo is published")
+	}
+
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshotCollection := debian.NewSnapshotCollection(context.database)
+		snapshots := context.CollectionFactory().SnapshotCollection().ByLocalRepoSource(repo)
-		snapshots := snapshotCollection.ByLocalRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Local repo `%s` was used to create following snapshots:\n", repo.Name)
@@ -37,7 +48,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = localRepoCollection.Drop(repo)
+	err = context.CollectionFactory().LocalRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
@@ -53,8 +64,8 @@ func makeCmdRepoDrop() *commander.Command {
 		UsageLine: "drop <name>",
 		Short:     "delete local repository",
 		Long: `
-Drop deletes information about local repo. Package data is not deleted
+Drop information about deletions from local repo. Package data is not deleted
-(it could be still used by other mirrors or snapshots).
+(since it could be still used by other mirrors or snapshots).
 
 Example:
 

cmd/repo_edit.go

@@ -0,0 +1,68 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoEdit(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	if context.Flags().Lookup("comment").Value.String() != "" {
+		repo.Comment = context.Flags().Lookup("comment").Value.String()
+	}
+
+	if context.Flags().Lookup("distribution").Value.String() != "" {
+		repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
+	}
+
+	if context.Flags().Lookup("component").Value.String() != "" {
+		repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
+	}
+
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fmt.Printf("Local repo %s successfully updated.\n", repo)
+	return err
+}
+
+func makeCmdRepoEdit() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoEdit,
+		UsageLine: "edit <name>",
+		Short:     "edit properties of local repository",
+		Long: `
+Command edit allows one to change metadata of local repository:
+comment, default distribution and component.
+
+Example:
+
+  $ aptly repo edit -distribution=wheezy testing
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-edit", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
+	cmd.Flag.String("distribution", "", "default distribution when publishing")
+	cmd.Flag.String("component", "", "default component when publishing")
+
+	return cmd
+}

cmd/repo_import.go

@@ -1,17 +1,17 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
+	"github.com/smira/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/flag"
 )
 
 func makeCmdRepoImport() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "import <src-mirror> <dst-repo> <package-spec> ...",
+		UsageLine: "import <src-mirror> <dst-repo> <package-query> ...",
 		Short:     "import packages from mirror to local repository",
 		Long: `
-Command import looks up packages matching <package-spec> in mirror <src-mirror>
+Command import looks up packages matching <package-query> in mirror <src-mirror>
 and copies them to local repo <dst-repo>.
 
 Example:

cmd/repo_list.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
-	"github.com/smira/aptly/debian"
 	"sort"
 )
 
@@ -12,35 +11,49 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	if localRepoCollection.Len() > 0 {
+	repos := make([]string, context.CollectionFactory().LocalRepoCollection().Len())
-		fmt.Printf("List of mirrors:\n")
+	i := 0
-		repos := make([]string, localRepoCollection.Len())
+	context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		i := 0
+		if raw {
-		localRepoCollection.ForEach(func(repo *debian.LocalRepo) error {
+			repos[i] = repo.Name
-			err := localRepoCollection.LoadComplete(repo)
+		} else {
+			err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 			if err != nil {
 				return err
 			}
 
 			repos[i] = fmt.Sprintf(" * %s (packages: %d)", repo.String(), repo.NumPackages())
-			i++
-			return nil
-		})
-
-		sort.Strings(repos)
-		for _, repo := range repos {
-			fmt.Println(repo)
 		}
+		i++
+		return nil
+	})
 
-		fmt.Printf("\nTo get more information about local repository, run `aptly repo show <name>`.\n")
+	context.CloseDatabase()
+
+	sort.Strings(repos)
+
+	if raw {
+		for _, repo := range repos {
+			fmt.Printf("%s\n", repo)
+		}
 	} else {
-		fmt.Printf("No local repositories found, create one with `aptly repo create ...`.\n")
+		if len(repos) > 0 {
+			fmt.Printf("List of local repos:\n")
+			for _, repo := range repos {
+				fmt.Println(repo)
+			}
+
+			fmt.Printf("\nTo get more information about local repository, run `aptly repo show <name>`.\n")
+		} else {
+			fmt.Printf("No local repositories found, create one with `aptly repo create ...`.\n")
+		}
 	}
+
 	return err
 }
 
@@ -50,14 +63,15 @@ func makeCmdRepoList() *commander.Command {
 		UsageLine: "list",
 		Short:     "list local repositories",
 		Long: `
-List shows full list of local package repositories.
+List command shows full list of local package repositories.
 
 Example:
 
   $ aptly repo list
 `,
-		Flag: *flag.NewFlagSet("aptly-repo-list", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+
 	return cmd
 }

cmd/repo_move.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/aptly/query"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"sort"
 )
 
@@ -12,30 +13,28 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 3 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	command := cmd.Name()
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	dstRepo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[1])
-
-	dstRepo, err := localRepoCollection.ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	err = localRepoCollection.LoadComplete(dstRepo)
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(dstRepo)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
 	var (
-		srcRefList *debian.PackageRefList
+		srcRefList *deb.PackageRefList
-		srcRepo    *debian.LocalRepo
+		srcRepo    *deb.LocalRepo
 	)
 
 	if command == "copy" || command == "move" {
-		srcRepo, err = localRepoCollection.ByName(args[0])
+		srcRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -44,43 +43,42 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to %s: source and destination are the same", command)
 		}
 
-		err = localRepoCollection.LoadComplete(srcRepo)
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(srcRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
 		srcRefList = srcRepo.RefList()
 	} else if command == "import" {
-		repoCollection := debian.NewRemoteRepoCollection(context.database)
+		var srcRemoteRepo *deb.RemoteRepo
 
-		srcRepo, err := repoCollection.ByName(args[0])
+		srcRemoteRepo, err = context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		err = repoCollection.LoadComplete(srcRepo)
+		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(srcRemoteRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		if srcRepo.RefList() == nil {
+		if srcRemoteRepo.RefList() == nil {
 			return fmt.Errorf("unable to %s: mirror not updated", command)
 		}
 
-		srcRefList = srcRepo.RefList()
+		srcRefList = srcRemoteRepo.RefList()
 	} else {
 		panic("unexpected command")
 	}
 
-	context.progress.Printf("Loading packages...\n")
+	context.Progress().Printf("Loading packages...\n")
 
-	packageCollection := debian.NewPackageCollection(context.database)
+	dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
-	dstList, err := debian.NewPackageListFromRefList(dstRepo.RefList(), packageCollection, context.progress)
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	srcList, err := debian.NewPackageListFromRefList(srcRefList, packageCollection, context.progress)
+	srcList, err := deb.NewPackageListFromRefList(srcRefList, context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -89,14 +87,14 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	var architecturesList []string
 
-	withDeps := cmd.Flag.Lookup("with-deps").Value.Get().(bool)
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
 
 	if withDeps {
 		dstList.PrepareIndex()
 
 		// Calculate architectures
-		if len(context.architecturesList) > 0 {
+		if len(context.ArchitecturesList()) > 0 {
-			architecturesList = context.architecturesList
+			architecturesList = context.ArchitecturesList()
 		} else {
 			architecturesList = dstList.Architectures(false)
 		}
@@ -108,7 +106,15 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	toProcess, err := srcList.Filter(args[2:], withDeps, dstList, context.dependencyOptions, architecturesList)
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i := 0; i < len(args)-2; i++ {
+		queries[i], err = query.Parse(args[i+2])
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+	}
+
+	toProcess, err := srcList.Filter(queries, withDeps, dstList, context.DependencyOptions(), architecturesList)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
@@ -123,7 +129,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		verb = "imported"
 	}
 
-	err = toProcess.ForEach(func(p *debian.Package) error {
+	err = toProcess.ForEach(func(p *deb.Package) error {
 		err = dstList.Add(p)
 		if err != nil {
 			return err
@@ -132,27 +138,27 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		if command == "move" {
 			srcList.Remove(p)
 		}
-		context.progress.ColoredPrintf("@g[o]@| %s %s", p, verb)
+		context.Progress().ColoredPrintf("@g[o]@| %s %s", p, verb)
 		return nil
 	})
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	if cmd.Flag.Lookup("dry-run").Value.Get().(bool) {
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
-		context.progress.Printf("\nChanges not saved, as dry run has been requested.\n")
+		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
-		dstRepo.UpdateRefList(debian.NewPackageRefListFromPackageList(dstList))
+		dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
 
-		err = localRepoCollection.Update(dstRepo)
+		err = context.CollectionFactory().LocalRepoCollection().Update(dstRepo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
 
 		if command == "move" {
-			srcRepo.UpdateRefList(debian.NewPackageRefListFromPackageList(srcList))
+			srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
 
-			err = localRepoCollection.Update(srcRepo)
+			err = context.CollectionFactory().LocalRepoCollection().Update(srcRepo)
 			if err != nil {
 				return fmt.Errorf("unable to save: %s", err)
 			}
@@ -165,10 +171,10 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 func makeCmdRepoMove() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "move <src-name> <dst-name> <package-spec> ...",
+		UsageLine: "move <src-name> <dst-name> <package-query> ...",
 		Short:     "move packages between local repositories",
 		Long: `
-Command move moves packages matching <package-spec> from local repo
+Command move moves packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
 Example:

cmd/repo_remove.go

@@ -2,57 +2,64 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/aptly/query"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
-	repo, err := localRepoCollection.ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	err = localRepoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	context.progress.Printf("Loading packages...\n")
+	context.Progress().Printf("Loading packages...\n")
 
-	packageCollection := debian.NewPackageCollection(context.database)
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
-	list, err := debian.NewPackageListFromRefList(repo.RefList(), packageCollection, context.progress)
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
+	queries := make([]deb.PackageQuery, len(args)-1)
+	for i := 0; i < len(args)-1; i++ {
+		queries[i], err = query.Parse(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to remove: %s", err)
+		}
+	}
+
 	list.PrepareIndex()
-	toRemove, err := list.Filter(args[1:], false, nil, 0, nil)
+	toRemove, err := list.Filter(queries, false, nil, 0, nil)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	toRemove.ForEach(func(p *debian.Package) error {
+	toRemove.ForEach(func(p *deb.Package) error {
 		list.Remove(p)
-		context.progress.ColoredPrintf("@r[-]@| %s removed", p)
+		context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
 		return nil
 	})
 
-	if cmd.Flag.Lookup("dry-run").Value.Get().(bool) {
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
-		context.progress.Printf("\nChanges not saved, as dry run has been requested.\n")
+		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
-		repo.UpdateRefList(debian.NewPackageRefListFromPackageList(list))
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = localRepoCollection.Update(repo)
+		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
@@ -64,10 +71,10 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 func makeCmdRepoRemove() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoRemove,
-		UsageLine: "remove <name> <package-spec> ...",
+		UsageLine: "remove <name> <package-query> ...",
 		Short:     "remove packages from local repository",
 		Long: `
-Commands removes packages matching <package-spec> from local repository
+Commands removes packages matching <package-query> from local repository
 <name>. If removed packages are not referenced by other repos or
 snapshots, they can be removed completely (including files) by running
 'aptly db cleanup'.

cmd/repo_rename.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyRepoRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.LocalRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	repo, err = context.CollectionFactory().LocalRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().LocalRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: local repo %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nLocal repo %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdRepoRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames local repository",
+		Long: `
+Command changes name of the local repo. Local repo name should be unique.
+
+Example:
+
+  $ aptly repo rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/repo_search.go

@@ -0,0 +1,26 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdRepoSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> <package-query>",
+		Short:     "search repo for packages matching query",
+		Long: `
+Command search displays list of packages in local repository that match package query
+
+Example:
+
+    $ aptly repo search my-software '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+
+	return cmd
+}

cmd/repo_show.go

@@ -2,36 +2,36 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/flag"
-	"github.com/smira/aptly/debian"
 )
 
 func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
-	repo, err := localRepoCollection.ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = localRepoCollection.LoadComplete(repo)
+	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
 	fmt.Printf("Name: %s\n", repo.Name)
 	fmt.Printf("Comment: %s\n", repo.Comment)
+	fmt.Printf("Default Distribution: %s\n", repo.DefaultDistribution)
+	fmt.Printf("Default Component: %s\n", repo.DefaultComponent)
 	fmt.Printf("Number of packages: %d\n", repo.NumPackages())
 
-	withPackages := cmd.Flag.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		ListPackagesRefList(repo.RefList())
 	}
@@ -45,7 +45,7 @@ func makeCmdRepoShow() *commander.Command {
 		UsageLine: "show <name>",
 		Short:     "show details about local repository",
 		Long: `
-Show shows full information about local package repository.
+Show command shows full information about local package repository.
 
 ex:
   $ aptly repo show testing

cmd/run.go

@@ -0,0 +1,45 @@
+package cmd
+
+import (
+	"fmt"
+	ctx "github.com/smira/aptly/context"
+	"github.com/smira/commander"
+)
+
+// Run runs single command starting from root cmd with args, optionally initializing context
+func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode int) {
+	defer func() {
+		if r := recover(); r != nil {
+			fatal, ok := r.(*ctx.FatalError)
+			if !ok {
+				panic(r)
+			}
+			fmt.Println("ERROR:", fatal.Message)
+			returnCode = fatal.ReturnCode
+		}
+	}()
+
+	returnCode = 0
+
+	flags, args, err := cmd.ParseFlags(cmdArgs)
+	if err != nil {
+		ctx.Fatal(err)
+	}
+
+	if initContext {
+		err = InitContext(flags)
+		if err != nil {
+			ctx.Fatal(err)
+		}
+		defer ShutdownContext()
+	}
+
+	context.UpdateFlags(flags)
+
+	err = cmd.Dispatch(args)
+	if err != nil {
+		ctx.Fatal(err)
+	}
+
+	return
+}

cmd/serve.go

@@ -2,28 +2,32 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/aptly"
-	"github.com/gonuts/flag"
+	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/debian"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"net"
 	"net/http"
 	"os"
 	"sort"
+	"strings"
 )
 
 func aptlyServe(cmd *commander.Command, args []string) error {
 	var err error
 
-	publishedCollection := debian.NewPublishedRepoCollection(context.database)
+	if len(args) != 0 {
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
 
-	if publishedCollection.Len() == 0 {
+	if context.CollectionFactory().PublishedRepoCollection().Len() == 0 {
 		fmt.Printf("No published repositories, unable to serve.\n")
 		return nil
 	}
 
-	listen := cmd.Flag.Lookup("listen").Value.String()
+	listen := context.Flags().Lookup("listen").Value.String()
 
 	listenHost, listenPort, err := net.SplitHostPort(listen)
 
@@ -40,11 +44,11 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Serving published repositories, recommended apt sources list:\n\n")
 
-	sources := make(sort.StringSlice, 0, publishedCollection.Len())
+	sources := make(sort.StringSlice, 0, context.CollectionFactory().PublishedRepoCollection().Len())
-	published := make(map[string]*debian.PublishedRepo, publishedCollection.Len())
+	published := make(map[string]*deb.PublishedRepo, context.CollectionFactory().PublishedRepoCollection().Len())
 
-	err = publishedCollection.ForEach(func(repo *debian.PublishedRepo) error {
+	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		err := publishedCollection.LoadComplete(repo, snapshotCollection)
+		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
 		if err != nil {
 			return err
 		}
@@ -72,19 +76,20 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		}
 
 		fmt.Printf("# %s\ndeb http://%s:%s/%s %s %s\n",
-			repo, listenHost, listenPort, prefix, repo.Distribution, repo.Component)
+			repo, listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 
 		if utils.StrSliceHasItem(repo.Architectures, "source") {
 			fmt.Printf("deb-src http://%s:%s/%s %s %s\n",
-				listenHost, listenPort, prefix, repo.Distribution, repo.Component)
+				listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 		}
 	}
 
-	context.database.Close()
+	publicPath := context.GetPublishedStorage("").(aptly.LocalPublishedStorage).PublicPath()
+	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
 
-	err = http.ListenAndServe(listen, http.FileServer(http.Dir(context.publishedStorage.PublicPath())))
+	err = http.ListenAndServe(listen, http.FileServer(http.Dir(publicPath)))
 	if err != nil {
 		return fmt.Errorf("unable to serve: %s", err)
 	}

cmd/snapshot.go

@@ -1,8 +1,7 @@
 package cmd
 
 import (
-	"github.com/gonuts/commander"
+	"github.com/smira/commander"
-	"github.com/gonuts/flag"
 )
 
 func makeCmdSnapshot() *commander.Command {
@@ -18,7 +17,9 @@ func makeCmdSnapshot() *commander.Command {
 			makeCmdSnapshotDiff(),
 			makeCmdSnapshotMerge(),
 			makeCmdSnapshotDrop(),
+			makeCmdSnapshotRename(),
+			makeCmdSnapshotSearch(),
+			makeCmdSnapshotFilter(),
 		},
-		Flag: *flag.NewFlagSet("aptly-snapshot", flag.ExitOnError),
 	}
 }

cmd/snapshot_create.go

@@ -2,52 +2,58 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
-	"github.com/smira/aptly/debian"
 )
 
 func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 	var (
 		err      error
-		snapshot *debian.Snapshot
+		snapshot *deb.Snapshot
 	)
 
 	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" {
 		// aptly snapshot create snap from mirror mirror
+		var repo *deb.RemoteRepo
+
 		repoName, snapshotName := args[3], args[0]
 
-		repoCollection := debian.NewRemoteRepoCollection(context.database)
+		repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(repoName)
-		repo, err := repoCollection.ByName(repoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = repoCollection.LoadComplete(repo)
+		err = repo.CheckLock()
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		snapshot, err = debian.NewSnapshotFromRepository(snapshotName, repo)
+		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		snapshot, err = deb.NewSnapshotFromRepository(snapshotName, repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" {
 		// aptly snapshot create snap from repo repo
+		var repo *deb.LocalRepo
+
 		localRepoName, snapshotName := args[3], args[0]
 
-		localRepoCollection := debian.NewLocalRepoCollection(context.database)
+		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(localRepoName)
-		repo, err := localRepoCollection.ByName(localRepoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = localRepoCollection.LoadComplete(repo)
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		snapshot, err = debian.NewSnapshotFromLocalRepo(snapshotName, repo)
+		snapshot, err = deb.NewSnapshotFromLocalRepo(snapshotName, repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -55,17 +61,15 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		// aptly snapshot create snap empty
 		snapshotName := args[0]
 
-		packageList := debian.NewPackageList()
+		packageList := deb.NewPackageList()
 
-		snapshot = debian.NewSnapshotFromPackageList(snapshotName, nil, packageList, "Created as empty")
+		snapshot = deb.NewSnapshotFromPackageList(snapshotName, nil, packageList, "Created as empty")
 	} else {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	err = context.CollectionFactory().SnapshotCollection().Add(snapshot)
-
-	err = snapshotCollection.Add(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to add snapshot: %s", err)
 	}
@@ -97,7 +101,6 @@ Example:
 
   $ aptly snapshot create wheezy-main-today from mirror wheezy-main
 `,
-		Flag: *flag.NewFlagSet("aptly-snapshot-create", flag.ExitOnError),
 	}
 
 	return cmd

cmd/snapshot_diff.go

@@ -2,55 +2,51 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/flag"
-	"github.com/smira/aptly/debian"
 )
 
 func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	onlyMatching := cmd.Flag.Lookup("only-matching").Value.Get().(bool)
+	onlyMatching := context.Flags().Lookup("only-matching").Value.Get().(bool)
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	packageCollection := debian.NewPackageCollection(context.database)
 
 	// Load <name-a> snapshot
-	snapshotA, err := snapshotCollection.ByName(args[0])
+	snapshotA, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(snapshotA)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotA)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
 	// Load <name-b> snapshot
-	snapshotB, err := snapshotCollection.ByName(args[1])
+	snapshotB, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(snapshotB)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotB)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
 	// Calculate diff
-	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), packageCollection)
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
 	if err != nil {
 		return fmt.Errorf("unable to calculate diff: %s", err)
 	}
 
 	if len(diff) == 0 {
-		context.progress.Printf("Snapshots are identical.\n")
+		context.Progress().Printf("Snapshots are identical.\n")
 	} else {
-		context.progress.Printf("  Arch   | Package                                  | Version in A                             | Version in B\n")
+		context.Progress().Printf("  Arch   | Package                                  | Version in A                             | Version in B\n")
 		for _, pdiff := range diff {
 			if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) {
 				continue
@@ -84,7 +80,7 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 				}
 			}
 
-			context.progress.ColoredPrintf(code+" %-6s | %-40s | %-40s | %-40s", arch, pkg, verA, verB)
+			context.Progress().ColoredPrintf(code+" %-6s | %-40s | %-40s | %-40s", arch, pkg, verA, verB)
 		}
 	}
 

cmd/snapshot_drop.go

@@ -2,33 +2,30 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/flag"
-	"github.com/smira/aptly/debian"
 )
 
 func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
-	snapshot, err := snapshotCollection.ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	publishedRepoCollection := debian.NewPublishedRepoCollection(context.database)
+	published := context.CollectionFactory().PublishedRepoCollection().BySnapshot(snapshot)
-	published := publishedRepoCollection.BySnapshot(snapshot)
 
 	if len(published) > 0 {
 		fmt.Printf("Snapshot `%s` is published currently:\n", snapshot.Name)
 		for _, repo := range published {
-			err = publishedRepoCollection.LoadComplete(repo, snapshotCollection)
+			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
 			if err != nil {
 				return fmt.Errorf("unable to load published: %s", err)
 			}
@@ -38,9 +35,9 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: snapshot is published")
 	}
 
-	force := cmd.Flag.Lookup("force").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := snapshotCollection.BySnapshotSource(snapshot)
+		snapshots := context.CollectionFactory().SnapshotCollection().BySnapshotSource(snapshot)
 		if len(snapshots) > 0 {
 			fmt.Printf("Snapshot `%s` was used as a source in following snapshots:\n", snapshot.Name)
 			for _, snap := range snapshots {
@@ -51,7 +48,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = snapshotCollection.Drop(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().Drop(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
@@ -67,7 +64,7 @@ func makeCmdSnapshotDrop() *commander.Command {
 		UsageLine: "drop <name>",
 		Short:     "delete snapshot",
 		Long: `
-Drop removes information about snapshot. If snapshot is published,
+Drop removes information about a snapshot. If snapshot is published,
 it can't be dropped.
 
 Example:

cmd/snapshot_filter.go

@@ -0,0 +1,107 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"sort"
+	"strings"
+)
+
+func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 3 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
+
+	// Load <source> snapshot
+	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Convert snapshot to package list
+	context.Progress().Printf("Loading packages (%d)...\n", source.RefList().Len())
+	packageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	context.Progress().Printf("Building indexes...\n")
+	packageList.PrepareIndex()
+
+	// Calculate architectures
+	var architecturesList []string
+
+	if len(context.ArchitecturesList()) > 0 {
+		architecturesList = context.ArchitecturesList()
+	} else {
+		architecturesList = packageList.Architectures(false)
+	}
+
+	sort.Strings(architecturesList)
+
+	if len(architecturesList) == 0 && withDeps {
+		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+	}
+
+	// Initial queries out of arguments
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i, arg := range args[2:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+	}
+
+	// Filter with dependencies as requested
+	result, err := packageList.Filter(queries, withDeps, nil, context.DependencyOptions(), architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Create <destination> snapshot
+	destination := deb.NewSnapshotFromPackageList(args[1], []*deb.Snapshot{source}, result,
+		fmt.Sprintf("Filtered '%s', query was: '%s'", source.Name, strings.Join(args[2:], " ")))
+
+	err = context.CollectionFactory().SnapshotCollection().Add(destination)
+	if err != nil {
+		return fmt.Errorf("unable to create snapshot: %s", err)
+	}
+
+	context.Progress().Printf("\nSnapshot %s successfully filtered.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+
+	return err
+}
+
+func makeCmdSnapshotFilter() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotFilter,
+		UsageLine: "filter <source> <destination> <package-query> ...",
+		Short:     "filter packages in snapshot producing another snapshot",
+		Long: `
+Command filter does filtering in snapshot <source>, producing another
+snapshot <destination>. Packages could be specified simply
+as 'package-name' or as package queries.
+
+Example:
+
+    $ aptly snapshot filter wheezy-main wheezy-required 'Priorioty (required)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-filter", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependent packages as well")
+
+	return cmd
+}

cmd/snapshot_list.go

@@ -2,44 +2,47 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
-	"github.com/smira/aptly/debian"
-	"sort"
 )
 
 func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
 
-	if snapshotCollection.Len() > 0 {
+	collection := context.CollectionFactory().SnapshotCollection()
-		fmt.Printf("List of snapshots:\n")
 
-		snapshots := make([]string, snapshotCollection.Len())
+	if raw {
-
+		collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
-		i := 0
+			fmt.Printf("%s\n", snapshot.Name)
-		snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
-			snapshots[i] = snapshot.String()
-			i++
 			return nil
 		})
-
-		sort.Strings(snapshots)
-		for _, snapshot := range snapshots {
-			fmt.Printf(" * %s\n", snapshot)
-		}
-
-		fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
 	} else {
-		fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n")
+		if collection.Len() > 0 {
-	}
+			fmt.Printf("List of snapshots:\n")
-	return err
 
+			err = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+				fmt.Printf(" * %s\n", snapshot.String())
+				return nil
+			})
+
+			if err != nil {
+				return err
+			}
+
+			fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
+		} else {
+			fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n")
+		}
+	}
+
+	return err
 }
 
 func makeCmdSnapshotList() *commander.Command {
@@ -54,8 +57,10 @@ Example:
 
   $ aptly snapshot list
 `,
-		Flag: *flag.NewFlagSet("aptly-snapshot-list", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+	cmd.Flag.String("sort", "name", "display list in 'name' or creation 'time' order")
+
 	return cmd
 }

cmd/snapshot_merge.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
-	"github.com/smira/aptly/debian"
 	"strings"
 )
 
@@ -12,29 +11,39 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	sources := make([]*deb.Snapshot, len(args)-1)
-
-	sources := make([]*debian.Snapshot, len(args)-1)
 
 	for i := 0; i < len(args)-1; i++ {
-		sources[i], err = snapshotCollection.ByName(args[i+1])
+		sources[i], err = context.CollectionFactory().SnapshotCollection().ByName(args[i+1])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
 
-		err = snapshotCollection.LoadComplete(sources[i])
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(sources[i])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
 	}
 
-	result := sources[0].RefList()
+	latest := context.Flags().Lookup("latest").Value.Get().(bool)
+	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
 
+	if noRemove && latest {
+		return fmt.Errorf("-no-remove and -latest can't be specified together")
+	}
+
+	overrideMatching := !latest && !noRemove
+
+	result := sources[0].RefList()
 	for i := 1; i < len(sources); i++ {
-		result = result.Merge(sources[i].RefList(), true)
+		result = result.Merge(sources[i].RefList(), overrideMatching, false)
+	}
+
+	if latest {
+		result.FilterLatestRefs()
 	}
 
 	sourceDescription := make([]string, len(sources))
@@ -43,10 +52,10 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	}
 
 	// Create <destination> snapshot
-	destination := debian.NewSnapshotFromRefList(args[0], sources, result,
+	destination := deb.NewSnapshotFromRefList(args[0], sources, result,
 		fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
-	err = snapshotCollection.Add(destination)
+	err = context.CollectionFactory().SnapshotCollection().Add(destination)
 	if err != nil {
 		return fmt.Errorf("unable to create snapshot: %s", err)
 	}
@@ -62,18 +71,20 @@ func makeCmdSnapshotMerge() *commander.Command {
 		UsageLine: "merge <destination> <source> [<source>...]",
 		Short:     "merges snapshots",
 		Long: `
-Merge merges several <source> snapshots into one <destination> snapshot.
+Merge command merges several <source> snapshots into one <destination> snapshot.
-Merge happens from left to right. Packages with the same name-architecture
+Merge happens from left to right. By default, packages with the same
-pair are replaced during merge (package from latest snapshot on the list
+name-architecture pair are replaced during merge (package from latest snapshot
-wins). If run with only one source snapshot, merge copies <source> into
+on the list wins).  If run with only one source snapshot, merge copies <source> into
 <destination>.
 
 Example:
 
     $ aptly snapshot merge wheezy-w-backports wheezy-main wheezy-backports
 `,
-		Flag: *flag.NewFlagSet("aptly-snapshot-merge", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("latest", false, "use only the latest version of each package")
+	cmd.Flag.Bool("no-remove", false, "don't remove duplicate arch/name packages")
+
 	return cmd
 }

cmd/snapshot_pull.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/aptly/query"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 	"sort"
 	"strings"
 )
@@ -13,61 +14,59 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 4 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	noDeps := cmd.Flag.Lookup("no-deps").Value.Get().(bool)
+	noDeps := context.Flags().Lookup("no-deps").Value.Get().(bool)
-	noRemove := cmd.Flag.Lookup("no-remove").Value.Get().(bool)
+	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
-
+	allMatches := context.Flags().Lookup("all-matches").Value.Get().(bool)
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	packageCollection := debian.NewPackageCollection(context.database)
 
 	// Load <name> snapshot
-	snapshot, err := snapshotCollection.ByName(args[0])
+	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
 	// Load <source> snapshot
-	source, err := snapshotCollection.ByName(args[1])
+	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(source)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	context.progress.Printf("Dependencies would be pulled into snapshot:\n    %s\nfrom snapshot:\n    %s\nand result would be saved as new snapshot %s.\n",
+	context.Progress().Printf("Dependencies would be pulled into snapshot:\n    %s\nfrom snapshot:\n    %s\nand result would be saved as new snapshot %s.\n",
 		snapshot, source, args[2])
 
 	// Convert snapshot to package list
-	context.progress.Printf("Loading packages (%d)...\n", snapshot.RefList().Len()+source.RefList().Len())
+	context.Progress().Printf("Loading packages (%d)...\n", snapshot.RefList().Len()+source.RefList().Len())
-	packageList, err := debian.NewPackageListFromRefList(snapshot.RefList(), packageCollection, context.progress)
+	packageList, err := deb.NewPackageListFromRefList(snapshot.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	sourcePackageList, err := debian.NewPackageListFromRefList(source.RefList(), packageCollection, context.progress)
+	sourcePackageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	context.progress.Printf("Building indexes...\n")
+	context.Progress().Printf("Building indexes...\n")
 	packageList.PrepareIndex()
 	sourcePackageList.PrepareIndex()
 
 	// Calculate architectures
 	var architecturesList []string
 
-	if len(context.architecturesList) > 0 {
+	if len(context.ArchitecturesList()) > 0 {
-		architecturesList = context.architecturesList
+		architecturesList = context.ArchitecturesList()
 	} else {
 		architecturesList = packageList.Architectures(false)
 	}
@@ -78,90 +77,71 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
 	}
 
-	// Initial dependencies out of arguments
+	// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
-	initialDependencies := make([]debian.Dependency, len(args)-3)
+	var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
-	for i, arg := range args[3:] {
-		initialDependencies[i], err = debian.ParseDependency(arg)
-		if err != nil {
-			return fmt.Errorf("unable to parse argument: %s", err)
-		}
-	}
-
-	// Perform pull
 	for _, arch := range architecturesList {
-		dependencies := make([]debian.Dependency, len(initialDependencies), 128)
+		archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
-		for i := range dependencies {
-			dependencies[i] = initialDependencies[i]
-			dependencies[i].Architecture = arch
-		}
-
-		// Go over list of initial dependencies + list of dependencies found
-		for i := 0; i < len(dependencies); i++ {
-			dep := dependencies[i]
-
-			// Search for package that can satisfy dependencies
-			pkg := sourcePackageList.Search(dep)
-			if pkg == nil {
-				context.progress.ColoredPrintf("@y[!]@| @!Dependency %s can't be satisfied with source %s@|", &dep, source)
-				continue
-			}
-
-			if !noRemove {
-				// Remove all packages with the same name and architecture
-				for p := packageList.Search(debian.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}); p != nil; {
-					packageList.Remove(p)
-					context.progress.ColoredPrintf("@r[-]@| %s removed", p)
-					p = packageList.Search(debian.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name})
-				}
-			}
-
-			// Add new discovered package
-			packageList.Add(pkg)
-			context.progress.ColoredPrintf("@g[+]@| %s added", pkg)
-
-			if noDeps {
-				continue
-			}
-
-			// Find missing dependencies for single added package
-			pL := debian.NewPackageList()
-			pL.Add(pkg)
-
-			missing, err := pL.VerifyDependencies(context.dependencyOptions, []string{arch}, packageList, nil)
-			if err != nil {
-				context.progress.ColoredPrintf("@y[!]@| @!Error while verifying dependencies for pkg %s: %s@|", pkg, err)
-			}
-
-			// Append missing dependencies to the list of dependencies to satisfy
-			for _, misDep := range missing {
-				found := false
-				for _, d := range dependencies {
-					if d == misDep {
-						found = true
-						break
-					}
-				}
-
-				if !found {
-					dependencies = append(dependencies, misDep)
-				}
-			}
-		}
 	}
 
-	if cmd.Flag.Lookup("dry-run").Value.Get().(bool) {
+	// Initial queries out of arguments
-		context.progress.Printf("\nNot creating snapshot, as dry run was requested.\n")
+	queries := make([]deb.PackageQuery, len(args)-3)
+	for i, arg := range args[3:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+		// Add architecture filter
+		queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
+	}
+
+	// Filter with dependencies as requested
+	result, err := sourcePackageList.Filter(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+	result.PrepareIndex()
+
+	alreadySeen := map[string]bool{}
+
+	result.ForEachIndexed(func(pkg *deb.Package) error {
+		key := pkg.Architecture + "_" + pkg.Name
+		_, seen := alreadySeen[key]
+
+		// If we haven't seen such name-architecture pair and were instructed to remove, remove it
+		if !noRemove && !seen {
+			// Remove all packages with the same name and architecture
+			pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
+			for _, p := range pS {
+				packageList.Remove(p)
+				context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
+			}
+		}
+
+		// If !allMatches, add only first matching name-arch package
+		if !seen || allMatches {
+			packageList.Add(pkg)
+			context.Progress().ColoredPrintf("@g[+]@| %s added", pkg)
+		}
+
+		alreadySeen[key] = true
+
+		return nil
+	})
+	alreadySeen = nil
+
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
+		context.Progress().Printf("\nNot creating snapshot, as dry run was requested.\n")
 	} else {
 		// Create <destination> snapshot
-		destination := debian.NewSnapshotFromPackageList(args[2], []*debian.Snapshot{snapshot, source}, packageList,
+		destination := deb.NewSnapshotFromPackageList(args[2], []*deb.Snapshot{snapshot, source}, packageList,
 			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", snapshot.Name, source.Name, strings.Join(args[3:], " ")))
 
-		err = snapshotCollection.Add(destination)
+		err = context.CollectionFactory().SnapshotCollection().Add(destination)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		context.progress.Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+		context.Progress().Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
 	}
 	return err
 }
@@ -169,14 +149,14 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 func makeCmdSnapshotPull() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotPull,
-		UsageLine: "pull <name> <source> <destination> <package-name> ...",
+		UsageLine: "pull <name> <source> <destination> <package-query> ...",
 		Short:     "pull packages from another snapshot",
 		Long: `
-Command pull pulls new packages along with its dependencies to snapshot <name>
+Command pull pulls new packages along with its' dependencies to snapshot <name>
 from snapshot <source>. Pull can upgrade package version in <name> with
 versions from <source> following dependencies. New snapshot <destination>
-is created as result of this process. Packages could be specified simply
+is created as a result of this process. Packages could be specified simply
-as 'package-name' or as dependency 'package-name (>= version)'.
+as 'package-name' or as package queries.
 
 Example:
 
@@ -188,6 +168,7 @@ Example:
 	cmd.Flag.Bool("dry-run", false, "don't create destination snapshot, just show what would be pulled")
 	cmd.Flag.Bool("no-deps", false, "don't process dependencies, just pull listed packages")
 	cmd.Flag.Bool("no-remove", false, "don't remove other package versions when pulling package")
+	cmd.Flag.Bool("all-matches", false, "pull all the packages that satisfy the dependency version requirements")
 
 	return cmd
 }

cmd/snapshot_rename.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlySnapshotRename(cmd *commander.Command, args []string) error {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = context.CollectionFactory().SnapshotCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: snapshot %s already exists", newName)
+	}
+
+	snapshot.Name = newName
+	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nSnapshot %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdSnapshotRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames snapshot",
+		Long: `
+Command changes name of the snapshot. Snapshot name should be unique.
+
+Example:
+
+  $ aptly snapshot rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/snapshot_search.go

@@ -0,0 +1,129 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"sort"
+)
+
+func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+	command := cmd.Parent.Name()
+
+	var reflist *deb.PackageRefList
+
+	if command == "snapshot" {
+		snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = snapshot.RefList()
+	} else if command == "mirror" {
+		repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else if command == "repo" {
+		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else {
+		panic("unknown command")
+	}
+
+	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	list.PrepareIndex()
+
+	q, err := query.Parse(args[1])
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
+	architecturesList := []string{}
+
+	if withDeps {
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
+		} else {
+			architecturesList = list.Architectures(false)
+		}
+
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+		}
+	}
+
+	result, err := list.Filter([]deb.PackageQuery{q}, withDeps,
+		nil, context.DependencyOptions(), architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	if result.Len() == 0 {
+		return fmt.Errorf("no results")
+	}
+
+	result.ForEach(func(p *deb.Package) error {
+		context.Progress().Printf("%s\n", p)
+		return nil
+	})
+
+	return err
+}
+
+func makeCmdSnapshotSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> <package-query>",
+		Short:     "search snapshot for packages matching query",
+		Long: `
+Command search displays list of packages in snapshot that match package query
+
+Example:
+
+    $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-search", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+
+	return cmd
+}

cmd/snapshot_show.go

@@ -2,27 +2,25 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/commander"
-	"github.com/gonuts/flag"
+	"github.com/smira/flag"
-	"github.com/smira/aptly/debian"
 )
 
 func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
-	snapshot, err := snapshotCollection.ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = snapshotCollection.LoadComplete(snapshot)
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -32,7 +30,7 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Description: %s\n", snapshot.Description)
 	fmt.Printf("Number of packages: %d\n", snapshot.NumPackages())
 
-	withPackages := cmd.Flag.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		ListPackagesRefList(snapshot.RefList())
 	}
@@ -46,7 +44,7 @@ func makeCmdSnapshotShow() *commander.Command {
 		UsageLine: "show <name>",
 		Short:     "shows details about snapshot",
 		Long: `
-Command show displays full information about snapshot.
+Command show displays full information about a snapshot.
 
 Example:
 

cmd/snapshot_verify.go

@@ -2,9 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
+	"github.com/smira/aptly/deb"
-	"github.com/gonuts/flag"
+	"github.com/smira/commander"
-	"github.com/smira/aptly/debian"
 	"sort"
 )
 
@@ -12,40 +11,38 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	snapshots := make([]*deb.Snapshot, len(args))
-	packageCollection := debian.NewPackageCollection(context.database)
-
-	snapshots := make([]*debian.Snapshot, len(args))
 	for i := range snapshots {
-		snapshots[i], err = snapshotCollection.ByName(args[i])
+		snapshots[i], err = context.CollectionFactory().SnapshotCollection().ByName(args[i])
 		if err != nil {
 			return fmt.Errorf("unable to verify: %s", err)
 		}
 
-		err = snapshotCollection.LoadComplete(snapshots[i])
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshots[i])
 		if err != nil {
 			return fmt.Errorf("unable to verify: %s", err)
 		}
 	}
 
-	context.progress.Printf("Loading packages...\n")
+	context.Progress().Printf("Loading packages...\n")
 
-	packageList, err := debian.NewPackageListFromRefList(snapshots[0].RefList(), packageCollection, context.progress)
+	packageList, err := deb.NewPackageListFromRefList(snapshots[0].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	sourcePackageList := debian.NewPackageList()
+	sourcePackageList := deb.NewPackageList()
 	err = sourcePackageList.Append(packageList)
 	if err != nil {
 		fmt.Errorf("unable to merge sources: %s", err)
 	}
 
+	var pL *deb.PackageList
 	for i := 1; i < len(snapshots); i++ {
-		pL, err := debian.NewPackageListFromRefList(snapshots[i].RefList(), packageCollection, context.progress)
+		pL, err = deb.NewPackageListFromRefList(snapshots[i].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 		if err != nil {
 			fmt.Errorf("unable to load packages: %s", err)
 		}
@@ -60,8 +57,8 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 
 	var architecturesList []string
 
-	if len(context.architecturesList) > 0 {
+	if len(context.ArchitecturesList()) > 0 {
-		architecturesList = context.architecturesList
+		architecturesList = context.ArchitecturesList()
 	} else {
 		architecturesList = packageList.Architectures(true)
 	}
@@ -70,17 +67,17 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
 	}
 
-	context.progress.Printf("Verifying...\n")
+	context.Progress().Printf("Verifying...\n")
 
-	missing, err := packageList.VerifyDependencies(context.dependencyOptions, architecturesList, sourcePackageList, context.progress)
+	missing, err := packageList.VerifyDependencies(context.DependencyOptions(), architecturesList, sourcePackageList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to verify dependencies: %s", err)
 	}
 
 	if len(missing) == 0 {
-		context.progress.Printf("All dependencies are satisfied.\n")
+		context.Progress().Printf("All dependencies are satisfied.\n")
 	} else {
-		context.progress.Printf("Missing dependencies (%d):\n", len(missing))
+		context.Progress().Printf("Missing dependencies (%d):\n", len(missing))
 		deps := make([]string, len(missing))
 		i := 0
 		for _, dep := range missing {
@@ -91,7 +88,7 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 		sort.Strings(deps)
 
 		for _, dep := range deps {
-			context.progress.Printf("  %s\n", dep)
+			context.Progress().Printf("  %s\n", dep)
 		}
 	}
 
@@ -104,7 +101,7 @@ func makeCmdSnapshotVerify() *commander.Command {
 		UsageLine: "verify <name> [<source> ...]",
 		Short:     "verify dependencies in snapshot",
 		Long: `
-Verify does depenency resolution in snapshot <name>, possibly using additional
+Verify does dependency resolution in snapshot <name>, possibly using additional
 snapshots <source> as dependency sources. All unsatisfied dependencies are
 printed.
 
@@ -112,7 +109,6 @@ Example:
 
     $ aptly snapshot verify wheezy-main wheezy-contrib wheezy-non-free
 `,
-		Flag: *flag.NewFlagSet("aptly-snapshot-verify", flag.ExitOnError),
 	}
 
 	return cmd

cmd/task.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdTask() *commander.Command {
+	return &commander.Command{
+		UsageLine: "task",
+		Short:     "manage aptly tasks",
+		Subcommands: []*commander.Command{
+			makeCmdTaskRun(),
+		},
+	}
+}

cmd/task_run.go

@@ -0,0 +1,154 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mattn/go-shellwords"
+	"github.com/smira/commander"
+)
+
+func aptlyTaskRun(cmd *commander.Command, args []string) error {
+	var err error
+	var cmdList [][]string
+
+	if filename := cmd.Flag.Lookup("filename").Value.Get().(string); filename != "" {
+		var text string
+		cmdArgs := []string{}
+
+		var finfo os.FileInfo
+		if finfo, err = os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
+			return fmt.Errorf("no such file, %s\n", filename)
+		}
+
+		fmt.Print("Reading file...\n\n")
+
+		var file *os.File
+		file, err = os.Open(filename)
+
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		scanner := bufio.NewScanner(file)
+
+		for scanner.Scan() {
+			text = strings.TrimSpace(scanner.Text()) + ","
+			parsedArgs, _ := shellwords.Parse(text)
+			cmdArgs = append(cmdArgs, parsedArgs...)
+		}
+
+		if err = scanner.Err(); err != nil {
+			return err
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("the file is empty")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else if len(args) == 0 {
+		var text string
+		cmdArgs := []string{}
+
+		fmt.Println("Please enter one command per line and leave one blank when finished.")
+
+		reader := bufio.NewReader(os.Stdin)
+		for {
+			fmt.Printf("> ")
+			text, _ = reader.ReadString('\n')
+			if text == "\n" {
+				break
+			} else {
+				text = strings.TrimSpace(text) + ","
+				parsedArgs, _ := shellwords.Parse(text)
+				cmdArgs = append(cmdArgs, parsedArgs...)
+			}
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("nothing entered")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else {
+		cmdList = formatCommands(args)
+	}
+
+	commandErrored := false
+
+	for i, command := range cmdList {
+		if !commandErrored {
+			err := context.ReOpenDatabase()
+			if err != nil {
+				return fmt.Errorf("failed to reopen DB: %s", err)
+			}
+			context.Progress().ColoredPrintf("@g%d) [Running]: %s@!", (i + 1), strings.Join(command, " "))
+			context.Progress().ColoredPrintf("\n@yBegin command output: ----------------------------@!")
+			context.Progress().Flush()
+
+			returnCode := Run(RootCommand(), command, false)
+			if returnCode != 0 {
+				commandErrored = true
+			}
+			context.Progress().ColoredPrintf("\n@yEnd command output: ------------------------------@!")
+			CleanupContext()
+		} else {
+			context.Progress().ColoredPrintf("@r%d) [Skipping]: %s@!", (i + 1), strings.Join(command, " "))
+		}
+	}
+
+	if commandErrored {
+		err = fmt.Errorf("at least one command has reported an error")
+	}
+
+	return err
+}
+
+func formatCommands(args []string) [][]string {
+	var cmd []string
+	var cmdArray [][]string
+
+	for _, s := range args {
+		if sTrimmed := strings.TrimRight(s, ","); sTrimmed != s {
+			cmd = append(cmd, sTrimmed)
+			cmdArray = append(cmdArray, cmd)
+			cmd = []string{}
+		} else {
+			cmd = append(cmd, s)
+		}
+	}
+
+	if len(cmd) > 0 {
+		cmdArray = append(cmdArray, cmd)
+	}
+
+	return cmdArray
+}
+
+func makeCmdTaskRun() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyTaskRun,
+		UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
+		Short:     "run aptly tasks",
+		Long: `
+Command helps organise multiple aptly commands in one single aptly task, running as single thread.
+
+Example:
+
+	  $ aptly task run
+	  > repo create local
+	  > repo add local pkg1
+	  > publish repo local
+	  > serve
+	  >
+
+`,
+	}
+
+	cmd.Flag.String("filename", "", "specifies the filename that contains the commands to run")
+	return cmd
+}

cmd/version.go

@@ -2,12 +2,16 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/commander"
 )
 
 func aptlyVersion(cmd *commander.Command, args []string) error {
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
 	fmt.Printf("aptly version: %s\n", aptly.Version)
 	return nil
 }
@@ -23,6 +27,5 @@ Shows aptly version.
 ex:
   $ aptly version
 `,
-		Flag: *flag.NewFlagSet("aptly-version", flag.ExitOnError),
 	}
 }

console/progress.go

@@ -14,6 +14,8 @@ const (
 	codeHideProgress
 	codeStop
 	codeFlush
+	codeBarEnabled
+	codeBarDisabled
 )
 
 type printTask struct {
@@ -81,6 +83,8 @@ func (p *Progress) InitBar(count int64, isBytes bool) {
 			p.bar.SetUnits(pb.U_BYTES)
 			p.bar.ShowSpeed = true
 		}
+
+		p.queue <- printTask{code: codeBarEnabled}
 		p.bar.Start()
 	}
 }
@@ -91,6 +95,7 @@ func (p *Progress) ShutdownBar() {
 		return
 	}
 	p.bar.Finish()
+	p.queue <- printTask{code: codeBarDisabled}
 	p.bar = nil
 	p.queue <- printTask{code: codeHideProgress}
 }
@@ -128,19 +133,30 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
 		p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"}
 	} else {
 		// stip color marks
-		var prev rune
+		var inColorMark, inCurly bool
 		msg = strings.Map(func(r rune) rune {
-			if prev == '@' {
+			if inColorMark {
-				prev = 0
+				if inCurly {
-				if r == '@' {
+					if r == '}' {
-					return r
+						inCurly = false
+						inColorMark = false
+						return -1
+					}
+				} else {
+					if r == '{' {
+						inCurly = true
+					} else if r == '@' {
+						return '@'
+					} else {
+						inColorMark = false
+					}
 				}
 				return -1
 			}
-			prev = r
-			if r == '@' {
-				return -1
 
+			if r == '@' {
+				inColorMark = true
+				return -1
 			}
 
 			return r
@@ -151,9 +167,15 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
 }
 
 func (p *Progress) worker() {
+	hasBar := false
+
 	for {
 		task := <-p.queue
 		switch task.code {
+		case codeBarEnabled:
+			hasBar = true
+		case codeBarDisabled:
+			hasBar = false
 		case codePrint:
 			if p.barShown {
 				fmt.Print("\r\033[2K")
@@ -161,7 +183,7 @@ func (p *Progress) worker() {
 			}
 			fmt.Print(task.message)
 		case codeProgress:
-			if p.bar != nil {
+			if hasBar {
 				fmt.Print("\r" + task.message)
 				p.barShown = true
 			}

console/terminal.go

@@ -1,9 +1,7 @@
-// +build !freebsd
-
 package console
 
 import (
-	"code.google.com/p/go.crypto/ssh/terminal"
+	"golang.org/x/crypto/ssh/terminal"
 	"syscall"
 )
 

console/terminal_bsd.go

@@ -1,10 +0,0 @@
-// +build freebsd
-
-package console
-
-// RunningOnTerminal checks whether stdout is terminal
-//
-// Stub for FreeBSD, until in go1.3 terminal.IsTerminal would start working for FreeBSD
-func RunningOnTerminal() bool {
-	return false
-}

context/context.go

@@ -0,0 +1,490 @@
+// Package context provides single entry to all resources
+package context
+
+import (
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/console"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/files"
+	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/s3"
+	"github.com/smira/aptly/swift"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"strings"
+	"sync"
+	"time"
+)
+
+// AptlyContext is a common context shared by all commands
+type AptlyContext struct {
+	sync.Mutex
+
+	flags, globalFlags *flag.FlagSet
+	configLoaded       bool
+
+	progress          aptly.Progress
+	downloader        aptly.Downloader
+	database          database.Storage
+	packagePool       aptly.PackagePool
+	publishedStorages map[string]aptly.PublishedStorage
+	collectionFactory *deb.CollectionFactory
+	dependencyOptions int
+	architecturesList []string
+	// Debug features
+	fileCPUProfile *os.File
+	fileMemProfile *os.File
+	fileMemStats   *os.File
+}
+
+// Check interface
+var _ aptly.PublishedStorageProvider = &AptlyContext{}
+
+// FatalError is type for panicking to abort execution with non-zero
+// exit code and print meaningful explanation
+type FatalError struct {
+	ReturnCode int
+	Message    string
+}
+
+// Fatal panics and aborts execution with exit code 1
+func Fatal(err error) {
+	returnCode := 1
+	if err == commander.ErrFlagError || err == commander.ErrCommandError {
+		returnCode = 2
+	}
+	panic(&FatalError{ReturnCode: returnCode, Message: err.Error()})
+}
+
+// Config loads and returns current configuration
+func (context *AptlyContext) Config() *utils.ConfigStructure {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.config()
+}
+
+func (context *AptlyContext) config() *utils.ConfigStructure {
+	if !context.configLoaded {
+		var err error
+
+		configLocation := context.globalFlags.Lookup("config").Value.String()
+		if configLocation != "" {
+			err = utils.LoadConfig(configLocation, &utils.Config)
+
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			configLocations := []string{
+				filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
+				"/etc/aptly.conf",
+			}
+
+			for _, configLocation := range configLocations {
+				err = utils.LoadConfig(configLocation, &utils.Config)
+				if err == nil {
+					break
+				}
+				if !os.IsNotExist(err) {
+					Fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
+				}
+			}
+
+			if err != nil {
+				fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
+				utils.SaveConfig(configLocations[0], &utils.Config)
+			}
+		}
+
+		context.configLoaded = true
+
+	}
+	return &utils.Config
+}
+
+// LookupOption checks boolean flag with default (usually config) and command-line
+// setting
+func (context *AptlyContext) LookupOption(defaultValue bool, name string) (result bool) {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.lookupOption(defaultValue, name)
+}
+
+func (context *AptlyContext) lookupOption(defaultValue bool, name string) (result bool) {
+	result = defaultValue
+
+	if context.globalFlags.IsSet(name) {
+		result = context.globalFlags.Lookup(name).Value.Get().(bool)
+	}
+
+	return
+}
+
+// DependencyOptions calculates options related to dependecy handling
+func (context *AptlyContext) DependencyOptions() int {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.dependencyOptions == -1 {
+		context.dependencyOptions = 0
+		if context.lookupOption(context.config().DepFollowSuggests, "dep-follow-suggests") {
+			context.dependencyOptions |= deb.DepFollowSuggests
+		}
+		if context.lookupOption(context.config().DepFollowRecommends, "dep-follow-recommends") {
+			context.dependencyOptions |= deb.DepFollowRecommends
+		}
+		if context.lookupOption(context.config().DepFollowAllVariants, "dep-follow-all-variants") {
+			context.dependencyOptions |= deb.DepFollowAllVariants
+		}
+		if context.lookupOption(context.config().DepFollowSource, "dep-follow-source") {
+			context.dependencyOptions |= deb.DepFollowSource
+		}
+	}
+
+	return context.dependencyOptions
+}
+
+// ArchitecturesList returns list of architectures fixed via command line or config
+func (context *AptlyContext) ArchitecturesList() []string {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.architecturesList == nil {
+		context.architecturesList = context.config().Architectures
+		optionArchitectures := context.globalFlags.Lookup("architectures").Value.String()
+		if optionArchitectures != "" {
+			context.architecturesList = strings.Split(optionArchitectures, ",")
+		}
+	}
+
+	return context.architecturesList
+}
+
+// Progress creates or returns Progress object
+func (context *AptlyContext) Progress() aptly.Progress {
+	context.Lock()
+	defer context.Unlock()
+
+	return context._progress()
+}
+
+func (context *AptlyContext) _progress() aptly.Progress {
+	if context.progress == nil {
+		context.progress = console.NewProgress()
+		context.progress.Start()
+	}
+
+	return context.progress
+}
+
+// Downloader returns instance of current downloader
+func (context *AptlyContext) Downloader() aptly.Downloader {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.downloader == nil {
+		var downloadLimit int64
+		limitFlag := context.flags.Lookup("download-limit")
+		if limitFlag != nil {
+			downloadLimit = limitFlag.Value.Get().(int64)
+		}
+		if downloadLimit == 0 {
+			downloadLimit = context.config().DownloadLimit
+		}
+		context.downloader = http.NewDownloader(context.config().DownloadConcurrency,
+			downloadLimit*1024, context._progress())
+	}
+
+	return context.downloader
+}
+
+// DBPath builds path to database
+func (context *AptlyContext) DBPath() string {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.dbPath()
+}
+
+// DBPath builds path to database
+func (context *AptlyContext) dbPath() string {
+	return filepath.Join(context.config().RootDir, "db")
+}
+
+// Database opens and returns current instance of database
+func (context *AptlyContext) Database() (database.Storage, error) {
+	context.Lock()
+	defer context.Unlock()
+
+	return context._database()
+}
+
+func (context *AptlyContext) _database() (database.Storage, error) {
+	if context.database == nil {
+		var err error
+
+		context.database, err = database.OpenDB(context.dbPath())
+		if err != nil {
+			return nil, fmt.Errorf("can't open database: %s", err)
+		}
+	}
+
+	return context.database, nil
+}
+
+// CloseDatabase closes the db temporarily
+func (context *AptlyContext) CloseDatabase() error {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.database == nil {
+		return nil
+	}
+
+	return context.database.Close()
+}
+
+// ReOpenDatabase reopens the db after close
+func (context *AptlyContext) ReOpenDatabase() error {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.database == nil {
+		return nil
+	}
+
+	const MaxTries = 10
+	const Delay = 10 * time.Second
+
+	for try := 0; try < MaxTries; try++ {
+		err := context.database.ReOpen()
+		if err == nil || strings.Index(err.Error(), "resource temporarily unavailable") == -1 {
+			return err
+		}
+		context._progress().Printf("Unable to reopen database, sleeping %s\n", Delay)
+		<-time.After(Delay)
+	}
+
+	return fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
+}
+
+// CollectionFactory builds factory producing all kinds of collections
+func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.collectionFactory == nil {
+		db, err := context._database()
+		if err != nil {
+			Fatal(err)
+		}
+		context.collectionFactory = deb.NewCollectionFactory(db)
+	}
+
+	return context.collectionFactory
+}
+
+// PackagePool returns instance of PackagePool
+func (context *AptlyContext) PackagePool() aptly.PackagePool {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.packagePool == nil {
+		context.packagePool = files.NewPackagePool(context.config().RootDir)
+	}
+
+	return context.packagePool
+}
+
+// GetPublishedStorage returns instance of PublishedStorage
+func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
+	context.Lock()
+	defer context.Unlock()
+
+	publishedStorage, ok := context.publishedStorages[name]
+	if !ok {
+		if name == "" {
+			publishedStorage = files.NewPublishedStorage(context.config().RootDir)
+		} else if strings.HasPrefix(name, "s3:") {
+			params, ok := context.config().S3PublishRoots[name[3:]]
+			if !ok {
+				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
+			}
+
+			var err error
+			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
+				params.Region, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
+				params.EncryptionMethod, params.PlusWorkaround)
+			if err != nil {
+				Fatal(err)
+			}
+		} else if strings.HasPrefix(name, "swift:") {
+			params, ok := context.config().SwiftPublishRoots[name[6:]]
+			if !ok {
+				Fatal(fmt.Errorf("published Swift storage %v not configured", name[6:]))
+			}
+
+			var err error
+			publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password,
+				params.AuthURL, params.Tenant, params.TenantID, params.Container, params.Prefix)
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			Fatal(fmt.Errorf("unknown published storage format: %v", name))
+		}
+		context.publishedStorages[name] = publishedStorage
+	}
+
+	return publishedStorage
+}
+
+// UploadPath builds path to upload storage
+func (context *AptlyContext) UploadPath() string {
+	return filepath.Join(context.Config().RootDir, "upload")
+}
+
+// UpdateFlags sets internal copy of flags in the context
+func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) {
+	context.Lock()
+	defer context.Unlock()
+
+	context.flags = flags
+}
+
+// Flags returns current command flags
+func (context *AptlyContext) Flags() *flag.FlagSet {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.flags
+}
+
+// GlobalFlags returns flags passed to all commands
+func (context *AptlyContext) GlobalFlags() *flag.FlagSet {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.globalFlags
+}
+
+// Shutdown shuts context down
+func (context *AptlyContext) Shutdown() {
+	context.Lock()
+	defer context.Unlock()
+
+	if aptly.EnableDebug {
+		if context.fileMemProfile != nil {
+			pprof.WriteHeapProfile(context.fileMemProfile)
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+		if context.fileCPUProfile != nil {
+			pprof.StopCPUProfile()
+			context.fileCPUProfile.Close()
+			context.fileCPUProfile = nil
+		}
+		if context.fileMemProfile != nil {
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+	}
+	if context.database != nil {
+		context.database.Close()
+		context.database = nil
+	}
+	if context.downloader != nil {
+		context.downloader.Abort()
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
+// Cleanup does partial shutdown of context
+func (context *AptlyContext) Cleanup() {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.downloader != nil {
+		context.downloader.Shutdown()
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
+// NewContext initializes context with default settings
+func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
+	var err error
+
+	context := &AptlyContext{
+		flags:             flags,
+		globalFlags:       flags,
+		dependencyOptions: -1,
+		publishedStorages: map[string]aptly.PublishedStorage{},
+	}
+
+	if aptly.EnableDebug {
+		cpuprofile := flags.Lookup("cpuprofile").Value.String()
+		if cpuprofile != "" {
+			context.fileCPUProfile, err = os.Create(cpuprofile)
+			if err != nil {
+				return nil, err
+			}
+			pprof.StartCPUProfile(context.fileCPUProfile)
+		}
+
+		memprofile := flags.Lookup("memprofile").Value.String()
+		if memprofile != "" {
+			context.fileMemProfile, err = os.Create(memprofile)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		memstats := flags.Lookup("memstats").Value.String()
+		if memstats != "" {
+			interval := flags.Lookup("meminterval").Value.Get().(time.Duration)
+
+			context.fileMemStats, err = os.Create(memstats)
+			if err != nil {
+				return nil, err
+			}
+
+			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
+
+			go func() {
+				var stats runtime.MemStats
+
+				start := time.Now().UnixNano()
+
+				for {
+					runtime.ReadMemStats(&stats)
+					if context.fileMemStats != nil {
+						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
+							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
+						time.Sleep(interval)
+					} else {
+						break
+					}
+				}
+			}()
+		}
+	}
+
+	return context, nil
+}

database/leveldb.go

@@ -7,6 +7,8 @@ import (
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/filter"
 	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 // Errors for Storage
@@ -22,11 +24,14 @@ type Storage interface {
 	KeysByPrefix(prefix []byte) [][]byte
 	FetchByPrefix(prefix []byte) [][]byte
 	Close() error
+	ReOpen() error
 	StartBatch()
 	FinishBatch() error
+	CompactDB() error
 }
 
 type levelDB struct {
+	path  string
 	db    *leveldb.DB
 	batch *leveldb.Batch
 }
@@ -36,19 +41,42 @@ var (
 	_ Storage = &levelDB{}
 )
 
-// OpenDB opens (creates) LevelDB database
+func internalOpen(path string) (*leveldb.DB, error) {
-func OpenDB(path string) (Storage, error) {
 	o := &opt.Options{
 		Filter: filter.NewBloomFilter(10),
 	}
 
-	db, err := leveldb.OpenFile(path, o)
+	return leveldb.OpenFile(path, o)
+}
+
+// OpenDB opens (creates) LevelDB database
+func OpenDB(path string) (Storage, error) {
+	db, err := internalOpen(path)
 	if err != nil {
 		return nil, err
 	}
-	return &levelDB{db: db}, nil
+	return &levelDB{db: db, path: path}, nil
 }
 
+// RecoverDB recovers LevelDB database from corruption
+func RecoverDB(path string) error {
+	stor, err := storage.OpenFile(path)
+	if err != nil {
+		return err
+	}
+
+	db, err := leveldb.Recover(stor, nil)
+	if err != nil {
+		return err
+	}
+
+	db.Close()
+	stor.Close()
+
+	return nil
+}
+
+// Get key value from database
 func (l *levelDB) Get(key []byte) ([]byte, error) {
 	value, err := l.db.Get(key, nil)
 	if err != nil {
@@ -61,6 +89,7 @@ func (l *levelDB) Get(key []byte) ([]byte, error) {
 	return value, nil
 }
 
+// Put saves key to database, if key has the same value in DB already, it is not saved
 func (l *levelDB) Put(key []byte, value []byte) error {
 	if l.batch != nil {
 		l.batch.Put(key, value)
@@ -72,13 +101,14 @@ func (l *levelDB) Put(key []byte, value []byte) error {
 			return err
 		}
 	} else {
-		if bytes.Compare(old, value) == 0 {
+		if bytes.Equal(old, value) {
 			return nil
 		}
 	}
 	return l.db.Put(key, value, nil)
 }
 
+// Delete removes key from DB
 func (l *levelDB) Delete(key []byte) error {
 	if l.batch != nil {
 		l.batch.Delete(key)
@@ -87,6 +117,7 @@ func (l *levelDB) Delete(key []byte) error {
 	return l.db.Delete(key, nil)
 }
 
+// KeysByPrefix returns all keys that start with prefix
 func (l *levelDB) KeysByPrefix(prefix []byte) [][]byte {
 	result := make([][]byte, 0, 20)
 
@@ -103,6 +134,7 @@ func (l *levelDB) KeysByPrefix(prefix []byte) [][]byte {
 	return result
 }
 
+// FetchByPrefix returns all values with keys that start with prefix
 func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
 	result := make([][]byte, 0, 20)
 
@@ -119,10 +151,30 @@ func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
 	return result
 }
 
+// Close finishes DB work
 func (l *levelDB) Close() error {
-	return l.db.Close()
+	if l.db == nil {
+		return nil
+	}
+	err := l.db.Close()
+	l.db = nil
+	return err
 }
 
+// Reopen tries to re-open the database
+func (l *levelDB) ReOpen() error {
+	if l.db != nil {
+		return nil
+	}
+
+	var err error
+	l.db, err = internalOpen(l.path)
+	return err
+}
+
+// StartBatch starts batch processing of keys
+//
+// All subsequent Get, Put and Delete would work on batch
 func (l *levelDB) StartBatch() {
 	if l.batch != nil {
 		panic("batch already started")
@@ -130,6 +182,7 @@ func (l *levelDB) StartBatch() {
 	l.batch = new(leveldb.Batch)
 }
 
+// FinishBatch finalizes the batch, saving operations
 func (l *levelDB) FinishBatch() error {
 	if l.batch == nil {
 		panic("no batch")
@@ -138,3 +191,8 @@ func (l *levelDB) FinishBatch() error {
 	l.batch = nil
 	return err
 }
+
+// CompactDB compacts database by merging layers
+func (l *levelDB) CompactDB() error {
+	return l.db.CompactRange(util.Range{})
+}

database/leveldb_test.go

@@ -1,8 +1,9 @@
 package database
 
 import (
-	. "launchpad.net/gocheck"
 	"testing"
+
+	. "gopkg.in/check.v1"
 )
 
 // Launch gocheck tests
@@ -11,7 +12,8 @@ func Test(t *testing.T) {
 }
 
 type LevelDBSuite struct {
-	db Storage
+	path string
+	db   Storage
 }
 
 var _ = Suite(&LevelDBSuite{})
@@ -19,7 +21,8 @@ var _ = Suite(&LevelDBSuite{})
 func (s *LevelDBSuite) SetUpTest(c *C) {
 	var err error
 
-	s.db, err = OpenDB(c.MkDir())
+	s.path = c.MkDir()
+	s.db, err = OpenDB(s.path)
 	c.Assert(err, IsNil)
 }
 
@@ -28,6 +31,29 @@ func (s *LevelDBSuite) TearDownTest(c *C) {
 	c.Assert(err, IsNil)
 }
 
+func (s *LevelDBSuite) TestRecoverDB(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Check(err, IsNil)
+
+	err = s.db.Close()
+	c.Check(err, IsNil)
+
+	err = RecoverDB(s.path)
+	c.Check(err, IsNil)
+
+	s.db, err = OpenDB(s.path)
+	c.Check(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}
+
 func (s *LevelDBSuite) TestGetPut(c *C) {
 	var (
 		key   = []byte("key")
@@ -122,3 +148,31 @@ func (s *LevelDBSuite) TestBatch(c *C) {
 	s.db.StartBatch()
 	c.Check(func() { s.db.StartBatch() }, Panics, "batch already started")
 }
+
+func (s *LevelDBSuite) TestCompactDB(c *C) {
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+
+	c.Check(s.db.CompactDB(), IsNil)
+}
+
+func (s *LevelDBSuite) TestReOpen(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	err = s.db.Close()
+	c.Assert(err, IsNil)
+
+	err = s.db.ReOpen()
+	c.Assert(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}

deb/collections.go

@@ -0,0 +1,94 @@
+package deb
+
+import (
+	"github.com/smira/aptly/database"
+	"sync"
+)
+
+// CollectionFactory is a single place to generate all desired collections
+type CollectionFactory struct {
+	*sync.Mutex
+	db             database.Storage
+	packages       *PackageCollection
+	remoteRepos    *RemoteRepoCollection
+	snapshots      *SnapshotCollection
+	localRepos     *LocalRepoCollection
+	publishedRepos *PublishedRepoCollection
+}
+
+// NewCollectionFactory creates new factory
+func NewCollectionFactory(db database.Storage) *CollectionFactory {
+	return &CollectionFactory{Mutex: &sync.Mutex{}, db: db}
+}
+
+// PackageCollection returns (or creates) new PackageCollection
+func (factory *CollectionFactory) PackageCollection() *PackageCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
+	if factory.packages == nil {
+		factory.packages = NewPackageCollection(factory.db)
+	}
+
+	return factory.packages
+}
+
+// RemoteRepoCollection returns (or creates) new RemoteRepoCollection
+func (factory *CollectionFactory) RemoteRepoCollection() *RemoteRepoCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
+	if factory.remoteRepos == nil {
+		factory.remoteRepos = NewRemoteRepoCollection(factory.db)
+	}
+
+	return factory.remoteRepos
+}
+
+// SnapshotCollection returns (or creates) new SnapshotCollection
+func (factory *CollectionFactory) SnapshotCollection() *SnapshotCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
+	if factory.snapshots == nil {
+		factory.snapshots = NewSnapshotCollection(factory.db)
+	}
+
+	return factory.snapshots
+}
+
+// LocalRepoCollection returns (or creates) new LocalRepoCollection
+func (factory *CollectionFactory) LocalRepoCollection() *LocalRepoCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
+	if factory.localRepos == nil {
+		factory.localRepos = NewLocalRepoCollection(factory.db)
+	}
+
+	return factory.localRepos
+}
+
+// PublishedRepoCollection returns (or creates) new PublishedRepoCollection
+func (factory *CollectionFactory) PublishedRepoCollection() *PublishedRepoCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
+	if factory.publishedRepos == nil {
+		factory.publishedRepos = NewPublishedRepoCollection(factory.db)
+	}
+
+	return factory.publishedRepos
+}
+
+// Flush removes all references to collections, so that memory could be reclaimed
+func (factory *CollectionFactory) Flush() {
+	factory.Lock()
+	defer factory.Unlock()
+
+	factory.localRepos = nil
+	factory.snapshots = nil
+	factory.remoteRepos = nil
+	factory.publishedRepos = nil
+	factory.packages = nil
+}

deb/deb.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"archive/tar"
@@ -47,7 +47,7 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 					return nil, fmt.Errorf("unable to read .tar archive: %s", err)
 				}
 
-				if tarHeader.Name == "./control" {
+				if tarHeader.Name == "./control" || tarHeader.Name == "control" {
 					reader := NewControlFileReader(untar)
 					stanza, err := reader.ReadStanza()
 					if err != nil {

deb/deb_test.go

@@ -1,10 +1,11 @@
-package debian
+package deb
 
 import (
 	"github.com/smira/aptly/utils"
-	. "launchpad.net/gocheck"
 	"path/filepath"
 	"runtime"
+
+	. "gopkg.in/check.v1"
 )
 
 type DebSuite struct {

deb/debian.go

@@ -0,0 +1,2 @@
+// Package deb implements Debian-specific repository handling
+package deb

deb/debian_test.go

@@ -1,8 +1,9 @@
-package debian
+package deb
 
 import (
-	. "launchpad.net/gocheck"
 	"testing"
+
+	. "gopkg.in/check.v1"
 )
 
 // Launch gocheck tests

deb/format.go

@@ -1,18 +1,87 @@
-package debian
+package deb
 
 import (
 	"bufio"
 	"errors"
 	"io"
 	"strings"
+	"unicode"
 )
 
 // Stanza or paragraph of Debian control file
 type Stanza map[string]string
 
 // Canonical order of fields in stanza
-var canocialOrder = []string{"Origin", "Label", "Suite", "Package", "Version", "Installed-Size", "Priority", "Section", "Maintainer",
+// Taken from: http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/vivid/apt/vivid/view/head:/apt-pkg/tagfile.cc#L504
-	"Architecture", "Codename", "Date", "Architectures", "Components", "Description", "MD5sum", "MD5Sum", "SHA1", "SHA256"}
+var (
+	canonicalOrderRelease = []string{
+		"Origin",
+		"Label",
+		"Archive",
+		"Suite",
+		"Version",
+		"Codename",
+		"Date",
+		"Architectures",
+		"Architecture",
+		"Components",
+		"Component",
+		"Description",
+		"MD5Sum",
+		"SHA1",
+		"SHA256",
+	}
+
+	canonicalOrderBinary = []string{
+		"Package",
+		"Essential",
+		"Status",
+		"Priority",
+		"Section",
+		"Installed-Size",
+		"Maintainer",
+		"Original-Maintainer",
+		"Architecture",
+		"Source",
+		"Version",
+		"Replaces",
+		"Provides",
+		"Depends",
+		"Pre-Depends",
+		"Recommends",
+		"Suggests",
+		"Conflicts",
+		"Breaks",
+		"Conffiles",
+		"Filename",
+		"Size",
+		"MD5Sum",
+		"MD5sum",
+		"SHA1",
+		"SHA256",
+		"Description",
+	}
+
+	canonicalOrderSource = []string{
+		"Package",
+		"Source",
+		"Binary",
+		"Version",
+		"Priority",
+		"Section",
+		"Maintainer",
+		"Original-Maintainer",
+		"Build-Depends",
+		"Build-Depends-Indep",
+		"Build-Conflicts",
+		"Build-Conflicts-Indep",
+		"Architecture",
+		"Standards-Version",
+		"Format",
+		"Directory",
+		"Files",
+	}
+)
 
 // Copy returns copy of Stanza
 func (s Stanza) Copy() (result Stanza) {
@@ -40,8 +109,16 @@ func writeField(w *bufio.Writer, field, value string) (err error) {
 }
 
 // WriteTo saves stanza back to stream, modifying itself on the fly
-func (s Stanza) WriteTo(w *bufio.Writer) error {
+func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
-	for _, field := range canocialOrder {
+	canonicalOrder := canonicalOrderBinary
+	if isSource {
+		canonicalOrder = canonicalOrderSource
+	}
+	if isRelease {
+		canonicalOrder = canonicalOrderRelease
+	}
+
+	for _, field := range canonicalOrder {
 		value, ok := s[field]
 		if ok {
 			delete(s, field)
@@ -81,6 +158,35 @@ func init() {
 	multilineFields["MD5Sum"] = true
 }
 
+func canonicalCase(field string) string {
+	upper := strings.ToUpper(field)
+	switch upper {
+	case "SHA1", "SHA256", "SHA512":
+		return upper
+	case "MD5SUM":
+		return "MD5Sum"
+	case "NOTAUTOMATIC":
+		return "NotAutomatic"
+	case "BUTAUTOMATICUPGRADES":
+		return "ButAutomaticUpgrades"
+	}
+
+	startOfWord := true
+
+	return strings.Map(func(r rune) rune {
+		if startOfWord {
+			startOfWord = false
+			return unicode.ToUpper(r)
+		}
+
+		if r == '-' {
+			startOfWord = true
+		}
+
+		return unicode.ToLower(r)
+	}, field)
+}
+
 // ControlFileReader implements reading of control files stanza by stanza
 type ControlFileReader struct {
 	scanner *bufio.Scanner
@@ -119,7 +225,7 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 			if len(parts) != 2 {
 				return nil, ErrMalformedStanza
 			}
-			lastField = parts[0]
+			lastField = canonicalCase(parts[0])
 			_, lastFieldMultiline = multilineFields[lastField]
 			if lastFieldMultiline {
 				stanza[lastField] = parts[1]

deb/format_test.go

@@ -1,10 +1,11 @@
-package debian
+package deb
 
 import (
 	"bufio"
 	"bytes"
-	. "launchpad.net/gocheck"
 	"strings"
+
+	. "gopkg.in/check.v1"
 )
 
 type ControlFileSuite struct {
@@ -107,7 +108,7 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 
 	buf := &bytes.Buffer{}
 	w := bufio.NewWriter(buf)
-	err = stanza.Copy().WriteTo(w)
+	err = stanza.Copy().WriteTo(w, false, false)
 	c.Assert(err, IsNil)
 	err = w.Flush()
 	c.Assert(err, IsNil)
@@ -122,6 +123,18 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 	c.Assert(strings.HasPrefix(str, "Package: "), Equals, true)
 }
 
+func (s *ControlFileSuite) TestCanonicalCase(c *C) {
+	c.Check(canonicalCase("Package"), Equals, "Package")
+	c.Check(canonicalCase("package"), Equals, "Package")
+	c.Check(canonicalCase("pAckaGe"), Equals, "Package")
+	c.Check(canonicalCase("MD5Sum"), Equals, "MD5Sum")
+	c.Check(canonicalCase("SHA1"), Equals, "SHA1")
+	c.Check(canonicalCase("SHA256"), Equals, "SHA256")
+	c.Check(canonicalCase("Package-List"), Equals, "Package-List")
+	c.Check(canonicalCase("package-list"), Equals, "Package-List")
+	c.Check(canonicalCase("packaGe-lIst"), Equals, "Package-List")
+}
+
 func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
 	for i := 0; i < c.N; i++ {
 		reader := bytes.NewBufferString(controlFile)

deb/graph.go

@@ -0,0 +1,120 @@
+package deb
+
+import (
+	"code.google.com/p/gographviz"
+	"fmt"
+	"strings"
+)
+
+// BuildGraph generates graph contents from aptly object database
+func BuildGraph(collectionFactory *CollectionFactory) (gographviz.Interface, error) {
+	var err error
+
+	graph := gographviz.NewEscape()
+	graph.SetDir(true)
+	graph.SetName("aptly")
+
+	existingNodes := map[string]bool{}
+
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *RemoteRepo) error {
+		err := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+
+		graph.AddNode("aptly", repo.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "darkgoldenrod1",
+			"label": fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
+				repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "),
+				strings.Join(repo.Architectures, ", "), repo.NumPackages()),
+		})
+		existingNodes[repo.UUID] = true
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *LocalRepo) error {
+		err := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+
+		graph.AddNode("aptly", repo.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "mediumseagreen",
+			"label": fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
+				repo.Name, repo.Comment, repo.NumPackages()),
+		})
+		existingNodes[repo.UUID] = true
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
+		existingNodes[snapshot.UUID] = true
+		return nil
+	})
+
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
+		err := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+
+		description := snapshot.Description
+		if snapshot.SourceKind == "repo" {
+			description = "Snapshot from repo"
+		}
+
+		graph.AddNode("aptly", snapshot.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "cadetblue1",
+			"label":     fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages()),
+		})
+
+		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
+			for _, uuid := range snapshot.SourceIDs {
+				_, exists := existingNodes[uuid]
+				if exists {
+					graph.AddEdge(uuid, snapshot.UUID, true, nil)
+				}
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	collectionFactory.PublishedRepoCollection().ForEach(func(repo *PublishedRepo) error {
+		graph.AddNode("aptly", repo.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "darkolivegreen1",
+			"label": fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution,
+				strings.Join(repo.Components(), " "), strings.Join(repo.Architectures, ", ")),
+		})
+
+		for _, uuid := range repo.Sources {
+			_, exists := existingNodes[uuid]
+			if exists {
+				graph.AddEdge(uuid, repo.UUID, true, nil)
+			}
+		}
+
+		return nil
+	})
+
+	return graph, nil
+}

deb/import.go

@@ -0,0 +1,181 @@
+package deb
+
+import (
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// CollectPackageFiles walks filesystem collecting all candidates for package files
+func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, failedFiles []string, err error) {
+	for _, location := range locations {
+		info, err2 := os.Stat(location)
+		if err2 != nil {
+			reporter.Warning("Unable to process %s: %s", location, err2)
+			failedFiles = append(failedFiles, location)
+			continue
+		}
+		if info.IsDir() {
+			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
+				if err3 != nil {
+					return err3
+				}
+				if info.IsDir() {
+					return nil
+				}
+
+				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+					strings.HasSuffix(info.Name(), ".dsc") {
+					packageFiles = append(packageFiles, path)
+				}
+
+				return nil
+			})
+		} else {
+			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+				strings.HasSuffix(info.Name(), ".dsc") {
+				packageFiles = append(packageFiles, location)
+			} else {
+				reporter.Warning("Unknown file extension: %s", location)
+				failedFiles = append(failedFiles, location)
+				continue
+			}
+		}
+	}
+
+	sort.Strings(packageFiles)
+
+	return
+}
+
+// ImportPackageFiles imports files into local repository
+func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier utils.Verifier,
+	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter) (processedFiles []string, failedFiles []string, err error) {
+	if forceReplace {
+		list.PrepareIndex()
+	}
+
+	for _, file := range packageFiles {
+		var (
+			stanza Stanza
+			p      *Package
+		)
+
+		candidateProcessedFiles := []string{}
+		isSourcePackage := strings.HasSuffix(file, ".dsc")
+		isUdebPackage := strings.HasSuffix(file, ".udeb")
+
+		if isSourcePackage {
+			stanza, err = GetControlFileFromDsc(file, verifier)
+
+			if err == nil {
+				stanza["Package"] = stanza["Source"]
+				delete(stanza, "Source")
+
+				p, err = NewSourcePackageFromControlFile(stanza)
+			}
+		} else {
+			stanza, err = GetControlFileFromDeb(file)
+			if isUdebPackage {
+				p = NewUdebPackageFromControlFile(stanza)
+			} else {
+				p = NewPackageFromControlFile(stanza)
+			}
+		}
+		if err != nil {
+			reporter.Warning("Unable to read file %s: %s", file, err)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		if p.Name == "" {
+			reporter.Warning("Empty package name on %s", file)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		if p.Version == "" {
+			reporter.Warning("Empty version on %s", file)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		if p.Architecture == "" {
+			reporter.Warning("Empty architecture on %s", file)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		var checksums utils.ChecksumInfo
+		checksums, err = utils.ChecksumsForFile(file)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if isSourcePackage {
+			p.UpdateFiles(append(p.Files(), PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
+		} else {
+			p.UpdateFiles([]PackageFile{{Filename: filepath.Base(file), Checksums: checksums}})
+		}
+
+		err = pool.Import(file, checksums.MD5)
+		if err != nil {
+			reporter.Warning("Unable to import file %s into pool: %s", file, err)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		candidateProcessedFiles = append(candidateProcessedFiles, file)
+
+		// go over all files, except for the last one (.dsc/.deb itself)
+		for _, f := range p.Files() {
+			if filepath.Base(f.Filename) == filepath.Base(file) {
+				continue
+			}
+			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
+			err = pool.Import(sourceFile, f.Checksums.MD5)
+			if err != nil {
+				reporter.Warning("Unable to import file %s into pool: %s", sourceFile, err)
+				failedFiles = append(failedFiles, file)
+				break
+			}
+
+			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
+		}
+		if err != nil {
+			// some files haven't been imported
+			continue
+		}
+
+		err = collection.Update(p)
+		if err != nil {
+			reporter.Warning("Unable to save package %s: %s", p, err)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		if forceReplace {
+			conflictingPackages := list.Search(Dependency{Pkg: p.Name, Version: p.Version, Relation: VersionEqual, Architecture: p.Architecture}, true)
+			for _, cp := range conflictingPackages {
+				reporter.Removed("%s removed due to conflict with package being added", cp)
+				list.Remove(cp)
+			}
+		}
+
+		err = list.Add(p)
+		if err != nil {
+			reporter.Warning("Unable to add package to repo %s: %s", p, err)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		reporter.Added("%s added", p)
+		processedFiles = append(processedFiles, candidateProcessedFiles...)
+	}
+
+	err = nil
+	return
+}

deb/index_files.go

@@ -0,0 +1,260 @@
+package deb
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type indexFiles struct {
+	publishedStorage aptly.PublishedStorage
+	basePath         string
+	renameMap        map[string]string
+	generatedFiles   map[string]utils.ChecksumInfo
+	tempDir          string
+	suffix           string
+	indexes          map[string]*indexFile
+}
+
+type indexFile struct {
+	parent       *indexFiles
+	discardable  bool
+	compressable bool
+	signable     bool
+	relativePath string
+	tempFilename string
+	tempFile     *os.File
+	w            *bufio.Writer
+}
+
+func (file *indexFile) BufWriter() (*bufio.Writer, error) {
+	if file.w == nil {
+		var err error
+		file.tempFilename = filepath.Join(file.parent.tempDir, strings.Replace(file.relativePath, "/", "_", -1))
+		file.tempFile, err = os.Create(file.tempFilename)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create temporary index file: %s", err)
+		}
+
+		file.w = bufio.NewWriter(file.tempFile)
+	}
+
+	return file.w, nil
+}
+
+func (file *indexFile) Finalize(signer utils.Signer) error {
+	if file.w == nil {
+		if file.discardable {
+			return nil
+		}
+		file.BufWriter()
+	}
+
+	err := file.w.Flush()
+	if err != nil {
+		file.tempFile.Close()
+		return fmt.Errorf("unable to write to index file: %s", err)
+	}
+
+	if file.compressable {
+		err = utils.CompressFile(file.tempFile)
+		if err != nil {
+			file.tempFile.Close()
+			return fmt.Errorf("unable to compress index file: %s", err)
+		}
+	}
+
+	file.tempFile.Close()
+
+	exts := []string{""}
+	if file.compressable {
+		exts = append(exts, ".gz", ".bz2")
+	}
+
+	for _, ext := range exts {
+		var checksumInfo utils.ChecksumInfo
+
+		checksumInfo, err = utils.ChecksumsForFile(file.tempFilename + ext)
+		if err != nil {
+			return fmt.Errorf("unable to collect checksums: %s", err)
+		}
+		file.parent.generatedFiles[file.relativePath+ext] = checksumInfo
+	}
+
+	err = file.parent.publishedStorage.MkDir(filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)))
+	if err != nil {
+		return fmt.Errorf("unable to create dir: %s", err)
+	}
+
+	for _, ext := range exts {
+		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext),
+			file.tempFilename+ext)
+		if err != nil {
+			return fmt.Errorf("unable to publish file: %s", err)
+		}
+
+		if file.parent.suffix != "" {
+			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] =
+				filepath.Join(file.parent.basePath, file.relativePath+ext)
+		}
+	}
+
+	if file.signable && signer != nil {
+		err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
+		if err != nil {
+			return fmt.Errorf("unable to detached sign file: %s", err)
+		}
+
+		err = signer.ClearSign(file.tempFilename, filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
+		if err != nil {
+			return fmt.Errorf("unable to clearsign file: %s", err)
+		}
+
+		if file.parent.suffix != "" {
+			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
+				filepath.Join(file.parent.basePath, file.relativePath+".gpg")
+			file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] =
+				filepath.Join(file.parent.basePath, "In"+file.relativePath)
+		}
+
+		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
+			file.tempFilename+".gpg")
+		if err != nil {
+			return fmt.Errorf("unable to publish file: %s", err)
+		}
+
+		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix),
+			filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
+		if err != nil {
+			return fmt.Errorf("unable to publish file: %s", err)
+		}
+	}
+
+	return nil
+}
+
+func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string) *indexFiles {
+	return &indexFiles{
+		publishedStorage: publishedStorage,
+		basePath:         basePath,
+		renameMap:        make(map[string]string),
+		generatedFiles:   make(map[string]utils.ChecksumInfo),
+		tempDir:          tempDir,
+		suffix:           suffix,
+		indexes:          make(map[string]*indexFile),
+	}
+}
+
+func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
+	if arch == "source" {
+		udeb = false
+	}
+	key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb)
+	file, ok := files.indexes[key]
+	if !ok {
+		var relativePath string
+
+		if arch == "source" {
+			relativePath = filepath.Join(component, "source", "Sources")
+		} else {
+			if udeb {
+				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
+			} else {
+				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
+			}
+		}
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  false,
+			compressable: true,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
+func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexFile {
+	if arch == "source" {
+		udeb = false
+	}
+	key := fmt.Sprintf("ri-%s-%s-%v", component, arch, udeb)
+	file, ok := files.indexes[key]
+	if !ok {
+		var relativePath string
+
+		if arch == "source" {
+			relativePath = filepath.Join(component, "source", "Release")
+		} else {
+			if udeb {
+				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Release")
+			} else {
+				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Release")
+			}
+		}
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  udeb,
+			compressable: false,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
+func (files *indexFiles) ReleaseFile() *indexFile {
+	return &indexFile{
+		parent:       files,
+		discardable:  false,
+		compressable: false,
+		signable:     true,
+		relativePath: "Release",
+	}
+}
+
+func (files *indexFiles) FinalizeAll(progress aptly.Progress) (err error) {
+	if progress != nil {
+		progress.InitBar(int64(len(files.indexes)), false)
+		defer progress.ShutdownBar()
+	}
+
+	for _, file := range files.indexes {
+		err = file.Finalize(nil)
+		if err != nil {
+			return
+		}
+		if progress != nil {
+			progress.AddBar(1)
+		}
+	}
+
+	files.indexes = make(map[string]*indexFile)
+
+	return
+}
+
+func (files *indexFiles) RenameFiles() error {
+	var err error
+
+	for oldName, newName := range files.renameMap {
+		err = files.publishedStorage.RenameFile(oldName, newName)
+		if err != nil {
+			return fmt.Errorf("unable to rename: %s", err)
+		}
+	}
+
+	return nil
+}

deb/list.go

@@ -1,11 +1,10 @@
-package debian
+package deb
 
 import (
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
 	"sort"
-	"strings"
 )
 
 // Dependency options
@@ -39,9 +38,15 @@ type PackageList struct {
 	providesIndex map[string][]*Package
 }
 
+// PackageConflictError means that package can't be added to the list due to error
+type PackageConflictError struct {
+	error
+}
+
 // Verify interface
 var (
 	_ sort.Interface = &PackageList{}
+	_ PackageCatalog = &PackageList{}
 )
 
 // NewPackageList creates empty package list
@@ -63,9 +68,9 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 	}
 
 	err := reflist.ForEach(func(key []byte) error {
-		p, err := collection.ByKey(key)
+		p, err2 := collection.ByKey(key)
-		if err != nil {
+		if err2 != nil {
-			return fmt.Errorf("unable to load package with key %s: %s", key, err)
+			return fmt.Errorf("unable to load package with key %s: %s", key, err2)
 		}
 		if progress != nil {
 			progress.AddBar(1)
@@ -86,11 +91,11 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 
 // Add appends package to package list, additionally checking for uniqueness
 func (l *PackageList) Add(p *Package) error {
-	key := string(p.Key(""))
+	key := string(p.ShortKey(""))
 	existing, ok := l.packages[key]
 	if ok {
 		if !existing.Equals(p) {
-			return fmt.Errorf("conflict in package %s: %#v != %#v", p, existing, p)
+			return &PackageConflictError{fmt.Errorf("conflict in package %s", p)}
 		}
 		return nil
 	}
@@ -101,7 +106,7 @@ func (l *PackageList) Add(p *Package) error {
 			l.providesIndex[provides] = append(l.providesIndex[provides], p)
 		}
 
-		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= p.Name })
+		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.lessPackages(p, l.packagesIndex[j]) })
 
 		// insert p into l.packagesIndex in position i
 		l.packagesIndex = append(l.packagesIndex, nil)
@@ -123,6 +128,22 @@ func (l *PackageList) ForEach(handler func(*Package) error) error {
 	return err
 }
 
+// ForEachIndexed calls handler for each package in list in indexed order
+func (l *PackageList) ForEachIndexed(handler func(*Package) error) error {
+	if !l.indexed {
+		panic("list not indexed, can't iterate")
+	}
+
+	var err error
+	for _, p := range l.packagesIndex {
+		err = handler(p)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
 // Len returns number of packages in the list
 func (l *PackageList) Len() int {
 	return len(l.packages)
@@ -137,7 +158,7 @@ func (l *PackageList) Append(pl *PackageList) error {
 		existing, ok := l.packages[k]
 		if ok {
 			if !existing.Equals(p) {
-				return fmt.Errorf("conflict in package %s: %#v != %#v", p, existing, p)
+				return fmt.Errorf("conflict in package %s", p)
 			}
 		} else {
 			l.packages[k] = p
@@ -149,7 +170,7 @@ func (l *PackageList) Append(pl *PackageList) error {
 
 // Remove removes package from the list, and updates index when required
 func (l *PackageList) Remove(p *Package) {
-	delete(l.packages, string(p.Key("")))
+	delete(l.packages, string(p.ShortKey("")))
 	if l.indexed {
 		for _, provides := range p.Provides {
 			for i, pkg := range l.providesIndex[provides] {
@@ -189,6 +210,19 @@ func (l *PackageList) Architectures(includeSource bool) (result []string) {
 	return
 }
 
+// Strings builds list of strings with package keys
+func (l *PackageList) Strings() []string {
+	result := make([]string, l.Len())
+	i := 0
+
+	for _, p := range l.packages {
+		result[i] = string(p.Key(""))
+		i += 1
+	}
+
+	return result
+}
+
 // depSliceDeduplicate removes dups in slice of Dependencies
 func depSliceDeduplicate(s []Dependency) []Dependency {
 	l := len(s)
@@ -220,6 +254,7 @@ func depSliceDeduplicate(s []Dependency) []Dependency {
 //
 // Analysis would be peformed for each architecture, in specified sources
 func (l *PackageList) VerifyDependencies(options int, architectures []string, sources *PackageList, progress aptly.Progress) ([]Dependency, error) {
+	l.PrepareIndex()
 	missing := make([]Dependency, 0, 128)
 
 	if progress != nil {
@@ -229,7 +264,7 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 	for _, arch := range architectures {
 		cache := make(map[string]bool, 2048)
 
-		for _, p := range l.packages {
+		for _, p := range l.packagesIndex {
 			if progress != nil {
 				progress.AddBar(1)
 			}
@@ -247,7 +282,6 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 				variants = depSliceDeduplicate(variants)
 
 				variantsMissing := make([]Dependency, 0, len(variants))
-				missingCount := 0
 
 				for _, dep := range variants {
 					if dep.Architecture == "" {
@@ -255,35 +289,23 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 					}
 
 					hash := dep.Hash()
-					r, ok := cache[hash]
+					satisfied, ok := cache[hash]
-					if ok {
+					if !ok {
-						if !r {
+						satisfied = sources.Search(dep, false) != nil
-							missingCount++
+						cache[hash] = satisfied
-						}
-						continue
 					}
 
-					if sources.Search(dep) == nil {
+					if !satisfied && !ok {
 						variantsMissing = append(variantsMissing, dep)
-						missingCount++
+					}
-					} else {
+
-						cache[hash] = true
+					if satisfied && options&DepFollowAllVariants == 0 {
+						variantsMissing = nil
+						break
 					}
 				}
 
-				if options&DepFollowAllVariants == DepFollowAllVariants {
+				missing = append(missing, variantsMissing...)
-					missing = append(missing, variantsMissing...)
-					for _, dep := range variantsMissing {
-						cache[dep.Hash()] = false
-					}
-				} else {
-					if missingCount == len(variants) {
-						missing = append(missing, variantsMissing...)
-						for _, dep := range variantsMissing {
-							cache[dep.Hash()] = false
-						}
-					}
-				}
 			}
 		}
 	}
@@ -300,13 +322,29 @@ func (l *PackageList) Swap(i, j int) {
 	l.packagesIndex[i], l.packagesIndex[j] = l.packagesIndex[j], l.packagesIndex[i]
 }
 
-// Compare compares two names in lexographical order
+func (l *PackageList) lessPackages(iPkg, jPkg *Package) bool {
+	if iPkg.Name == jPkg.Name {
+		cmp := CompareVersions(iPkg.Version, jPkg.Version)
+		if cmp == 0 {
+			return iPkg.Architecture < jPkg.Architecture
+		}
+		return cmp == 1
+	}
+
+	return iPkg.Name < jPkg.Name
+}
+
+// Less compares two packages by name (lexographical) and version (latest to oldest)
 func (l *PackageList) Less(i, j int) bool {
-	return l.packagesIndex[i].Name < l.packagesIndex[j].Name
+	return l.lessPackages(l.packagesIndex[i], l.packagesIndex[j])
 }
 
 // PrepareIndex prepares list for indexing
 func (l *PackageList) PrepareIndex() {
+	if l.indexed {
+		return
+	}
+
 	l.packagesIndex = make([]*Package, l.Len())
 	l.providesIndex = make(map[string][]*Package, 128)
 
@@ -325,16 +363,49 @@ func (l *PackageList) PrepareIndex() {
 	l.indexed = true
 }
 
-// Search searches package index for specified package
+// Scan searches package index using full scan
-func (l *PackageList) Search(dep Dependency) *Package {
+func (l *PackageList) Scan(q PackageQuery) (result *PackageList) {
+	result = NewPackageList()
+	for _, pkg := range l.packages {
+		if q.Matches(pkg) {
+			result.Add(pkg)
+		}
+	}
+
+	return
+}
+
+// SearchSupported returns true for PackageList
+func (l *PackageList) SearchSupported() bool {
+	return true
+}
+
+// SearchByKey looks up package by exact key reference
+func (l *PackageList) SearchByKey(arch, name, version string) (result *PackageList) {
+	result = NewPackageList()
+
+	pkg := l.packages["P"+arch+" "+name+" "+version]
+	if pkg != nil {
+		result.Add(pkg)
+	}
+
+	return
+}
+
+// Search searches package index for specified package(s) using optimized queries
+func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*Package) {
 	if !l.indexed {
 		panic("list not indexed, can't search")
 	}
 
 	if dep.Relation == VersionDontCare {
 		for _, p := range l.providesIndex[dep.Pkg] {
-			if p.MatchesArchitecture(dep.Architecture) {
+			if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
-				return p
+				searchResults = append(searchResults, p)
+
+				if !allMatches {
+					break
+				}
 			}
 		}
 	}
@@ -344,16 +415,21 @@ func (l *PackageList) Search(dep Dependency) *Package {
 	for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
 		p := l.packagesIndex[i]
 		if p.MatchesDependency(dep) {
-			return p
+			searchResults = append(searchResults, p)
+
+			if !allMatches {
+				break
+			}
 		}
 
 		i++
 	}
-	return nil
+
+	return
 }
 
 // Filter filters package index by specified queries (ORed together), possibly pulling dependencies
-func (l *PackageList) Filter(queries []string, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
+func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
 	if !l.indexed {
 		panic("list not indexed, can't filter")
 	}
@@ -361,53 +437,17 @@ func (l *PackageList) Filter(queries []string, withDependencies bool, source *Pa
 	result := NewPackageList()
 
 	for _, query := range queries {
-		isDepQuery := strings.IndexAny(query, " (){}=<>") != -1
+		result.Append(query.Query(l))
-
-		if !isDepQuery {
-			// try to interpret query as package string representation
-
-			// convert Package.String() to Package.Key()
-			i := strings.Index(query, "_")
-			if i != -1 {
-				pkg, query := query[:i], query[i+1:]
-				j := strings.LastIndex(query, "_")
-				if j != -1 {
-					version, arch := query[:j], query[j+1:]
-					p := l.packages["P"+arch+" "+pkg+" "+version]
-					if p != nil {
-						result.Add(p)
-						continue
-					}
-				}
-			}
-		}
-
-		// try as dependency
-		dep, err := ParseDependency(query)
-		if err != nil {
-			if isDepQuery {
-				return nil, err
-			}
-			// parsing failed, but probably that wasn't a dep query
-			continue
-		}
-
-		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= dep.Pkg })
-
-		for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
-			p := l.packagesIndex[i]
-			if p.MatchesDependency(dep) {
-				result.Add(p)
-			}
-			i++
-		}
 	}
 
 	if withDependencies {
 		added := result.Len()
+		result.PrepareIndex()
 
 		dependencySource := NewPackageList()
-		dependencySource.Append(source)
+		if source != nil {
+			dependencySource.Append(source)
+		}
 		dependencySource.Append(result)
 		dependencySource.PrepareIndex()
 
@@ -423,11 +463,22 @@ func (l *PackageList) Filter(queries []string, withDependencies bool, source *Pa
 
 			// try to satisfy dependencies
 			for _, dep := range missing {
-				p := l.Search(dep)
+				// dependency might have already been satisfied
-				if p != nil {
+				// with packages already been added
-					result.Add(p)
+				if result.Search(dep, false) != nil {
-					dependencySource.Add(p)
+					continue
-					added++
+				}
+
+				searchResults := l.Search(dep, false)
+				if searchResults != nil {
+					for _, p := range searchResults {
+						result.Add(p)
+						dependencySource.Add(p)
+						added++
+						if dependencyOptions&DepFollowAllVariants == 0 {
+							break
+						}
+					}
 				}
 			}
 		}

deb/list_test.go

@@ -0,0 +1,426 @@
+package deb
+
+import (
+	"errors"
+	"regexp"
+	"sort"
+	"strings"
+
+	. "gopkg.in/check.v1"
+)
+
+type containsChecker struct {
+	*CheckerInfo
+}
+
+func (c *containsChecker) Check(params []interface{}, names []string) (result bool, error string) {
+	var (
+		pkgSlice1 []*Package
+		pkgSlice2 []*Package
+		ok        bool
+	)
+
+	pkgMap := make(map[*Package]bool)
+
+	pkgSlice1, ok = params[0].([]*Package)
+	if !ok {
+		return false, "The first parameter is not a Package slice"
+	}
+	pkgSlice2, ok = params[1].([]*Package)
+	if !ok {
+		return false, "The second parameter is not a Package slice"
+	}
+
+	for _, pkg := range pkgSlice2 {
+		pkgMap[pkg] = true
+	}
+
+	for _, pkg := range pkgSlice1 {
+		if _, ok := pkgMap[pkg]; !ok {
+			return false, ""
+		}
+	}
+	return true, ""
+}
+
+var Contains = &containsChecker{&CheckerInfo{Name: "Contains", Params: []string{"Container", "Expected to contain"}}}
+
+type PackageListSuite struct {
+	// Simple list with "real" packages from stanzas
+	list                   *PackageList
+	p1, p2, p3, p4, p5, p6 *Package
+
+	// Mocked packages in list
+	packages       []*Package
+	packages2      []*Package
+	sourcePackages []*Package
+	il             *PackageList
+	il2            *PackageList
+}
+
+var _ = Suite(&PackageListSuite{})
+
+func (s *PackageListSuite) SetUpTest(c *C) {
+	s.list = NewPackageList()
+
+	s.p1 = NewPackageFromControlFile(packageStanza.Copy())
+	s.p2 = NewPackageFromControlFile(packageStanza.Copy())
+	stanza := packageStanza.Copy()
+	stanza["Package"] = "mars-invaders"
+	s.p3 = NewPackageFromControlFile(stanza)
+	stanza = packageStanza.Copy()
+	stanza["Source"] = "unknown-planet"
+	s.p4 = NewPackageFromControlFile(stanza)
+	stanza = packageStanza.Copy()
+	stanza["Package"] = "lonely-strangers"
+	s.p5 = NewPackageFromControlFile(stanza)
+	stanza = packageStanza.Copy()
+	stanza["Version"] = "99.1"
+	s.p6 = NewPackageFromControlFile(stanza)
+
+	s.il = NewPackageList()
+	s.packages = []*Package{
+		{Name: "lib", Version: "1.0", Architecture: "i386", Source: "lib (0.9)", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"mail-agent"}}},
+		{Name: "dpkg", Version: "1.7", Architecture: "i386", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
+		{Name: "data", Version: "1.1~bp1", Architecture: "all", Source: "app", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
+		{Name: "app", Version: "1.1~bp1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "mailer", Version: "3.5.8", Architecture: "i386", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		{Name: "app", Version: "1.1~bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "1.1~bp1", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
+		{Name: "app", Version: "1.0", Architecture: "s390", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "aa", Version: "2.0-1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
+		{Name: "dpkg", Version: "1.6.1-3", Architecture: "amd64", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
+		{Name: "libx", Version: "1.5", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
+		{Name: "dpkg", Version: "1.6.1-3", Architecture: "arm", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
+		{Name: "dpkg", Version: "1.6.1-3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "dpkg", Version: "1.7", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+	}
+	for _, p := range s.packages {
+		s.il.Add(p)
+	}
+	s.il.PrepareIndex()
+
+	s.il2 = NewPackageList()
+	s.packages2 = []*Package{
+		{Name: "mailer", Version: "3.5.8", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		{Name: "sendmail", Version: "1.0", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		{Name: "app", Version: "1.1-bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "1.1-bp2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "1.2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
+		{Name: "app", Version: "3.0", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+	}
+	for _, p := range s.packages2 {
+		s.il2.Add(p)
+	}
+	s.il2.PrepareIndex()
+
+	s.sourcePackages = []*Package{
+		{Name: "postfix", Version: "1.3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "app", Version: "1.1~bp1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "aa", Version: "2.0-1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "lib", Version: "0.9", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+	}
+
+}
+
+func (s *PackageListSuite) TestAddLen(c *C) {
+	c.Check(s.list.Len(), Equals, 0)
+	c.Check(s.list.Add(s.p1), IsNil)
+	c.Check(s.list.Len(), Equals, 1)
+	c.Check(s.list.Add(s.p2), IsNil)
+	c.Check(s.list.Len(), Equals, 1)
+	c.Check(s.list.Add(s.p3), IsNil)
+	c.Check(s.list.Len(), Equals, 2)
+	c.Check(s.list.Add(s.p4), ErrorMatches, "conflict in package.*")
+}
+
+func (s *PackageListSuite) TestRemove(c *C) {
+	c.Check(s.list.Add(s.p1), IsNil)
+	c.Check(s.list.Add(s.p3), IsNil)
+	c.Check(s.list.Len(), Equals, 2)
+
+	s.list.Remove(s.p1)
+	c.Check(s.list.Len(), Equals, 1)
+}
+
+func (s *PackageListSuite) TestAddWhenIndexed(c *C) {
+	c.Check(s.list.Len(), Equals, 0)
+	s.list.PrepareIndex()
+
+	c.Check(s.list.Add(&Package{Name: "a1st", Version: "1.0", Architecture: "i386", Provides: []string{"fa", "fb"}}), IsNil)
+	c.Check(s.list.packagesIndex[0].Name, Equals, "a1st")
+	c.Check(s.list.providesIndex["fa"][0].Name, Equals, "a1st")
+	c.Check(s.list.providesIndex["fb"][0].Name, Equals, "a1st")
+
+	c.Check(s.list.Add(&Package{Name: "c3rd", Version: "1.0", Architecture: "i386", Provides: []string{"fa"}}), IsNil)
+	c.Check(s.list.packagesIndex[0].Name, Equals, "a1st")
+	c.Check(s.list.packagesIndex[1].Name, Equals, "c3rd")
+	c.Check(s.list.providesIndex["fa"][0].Name, Equals, "a1st")
+	c.Check(s.list.providesIndex["fa"][1].Name, Equals, "c3rd")
+	c.Check(s.list.providesIndex["fb"][0].Name, Equals, "a1st")
+
+	c.Check(s.list.Add(&Package{Name: "b2nd", Version: "1.0", Architecture: "i386"}), IsNil)
+	c.Check(s.list.packagesIndex[0].Name, Equals, "a1st")
+	c.Check(s.list.packagesIndex[1].Name, Equals, "b2nd")
+	c.Check(s.list.packagesIndex[2].Name, Equals, "c3rd")
+	c.Check(s.list.providesIndex["fa"][0].Name, Equals, "a1st")
+	c.Check(s.list.providesIndex["fa"][1].Name, Equals, "c3rd")
+	c.Check(s.list.providesIndex["fb"][0].Name, Equals, "a1st")
+}
+
+func (s *PackageListSuite) TestRemoveWhenIndexed(c *C) {
+	s.il.Remove(s.packages[0])
+	names := make([]string, s.il.Len())
+	for i, p := range s.il.packagesIndex {
+		names[i] = p.Name
+	}
+	c.Check(names, DeepEquals, []string{"aa", "app", "app", "app", "app", "data", "dpkg", "dpkg", "dpkg", "dpkg", "dpkg", "libx", "mailer"})
+
+	s.il.Remove(s.packages[4])
+	names = make([]string, s.il.Len())
+	for i, p := range s.il.packagesIndex {
+		names[i] = p.Name
+	}
+	c.Check(names, DeepEquals, []string{"aa", "app", "app", "app", "app", "data", "dpkg", "dpkg", "dpkg", "dpkg", "dpkg", "libx"})
+	c.Check(s.il.providesIndex["mail-agent"], DeepEquals, []*Package{})
+
+	s.il.Remove(s.packages[9])
+	names = make([]string, s.il.Len())
+	for i, p := range s.il.packagesIndex {
+		names[i] = p.Name
+	}
+	c.Check(names, DeepEquals, []string{"aa", "app", "app", "app", "app", "data", "dpkg", "dpkg", "dpkg", "dpkg", "libx"})
+	c.Check(s.il.providesIndex["package-installer"], HasLen, 2)
+
+	s.il.Remove(s.packages[1])
+	names = make([]string, s.il.Len())
+	for i, p := range s.il.packagesIndex {
+		names[i] = p.Name
+	}
+	c.Check(names, DeepEquals, []string{"aa", "app", "app", "app", "app", "data", "dpkg", "dpkg", "dpkg", "libx"})
+	c.Check(s.il.providesIndex["package-installer"], DeepEquals, []*Package{s.packages[11]})
+}
+
+func (s *PackageListSuite) TestForeach(c *C) {
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+
+	Len := 0
+	err := s.list.ForEach(func(*Package) error {
+		Len++
+		return nil
+	})
+
+	c.Check(Len, Equals, 2)
+	c.Check(err, IsNil)
+
+	e := errors.New("a")
+
+	err = s.list.ForEach(func(*Package) error {
+		return e
+	})
+
+	c.Check(err, Equals, e)
+
+}
+
+func (s *PackageListSuite) TestIndex(c *C) {
+	c.Check(len(s.il.providesIndex), Equals, 2)
+	c.Check(len(s.il.providesIndex["mail-agent"]), Equals, 1)
+	c.Check(len(s.il.providesIndex["package-installer"]), Equals, 3)
+	c.Check(s.il.packagesIndex[0], Equals, s.packages[8])
+}
+
+func (s *PackageListSuite) TestAppend(c *C) {
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+
+	err := s.list.Append(s.il)
+	c.Check(err, IsNil)
+	c.Check(s.list.Len(), Equals, 16)
+
+	list := NewPackageList()
+	list.Add(s.p4)
+
+	err = s.list.Append(list)
+	c.Check(err, ErrorMatches, "conflict.*")
+
+	s.list.PrepareIndex()
+	c.Check(func() { s.list.Append(s.il) }, Panics, "Append not supported when indexed")
+}
+
+func (s *PackageListSuite) TestSearch(c *C) {
+	//allMatches = False
+	c.Check(func() { s.list.Search(Dependency{Architecture: "i386", Pkg: "app"}, false) }, Panics, "list not indexed, can't search")
+
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "mail-agent"}, false), DeepEquals, []*Package{s.packages[4]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "puppy"}, false), IsNil)
+
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionEqual, Version: "1.1~bp2"}, false), IsNil)
+
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLess, Version: "1.1~~"}, false), IsNil)
+
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1~~"}, false), IsNil)
+
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.0"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreater, Version: "1.2"}, false), IsNil)
+
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}, false), DeepEquals, []*Package{s.packages[3]})
+	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}, false), IsNil)
+
+	// search w/o version should return package with latest version
+	c.Check(s.il.Search(Dependency{Architecture: "source", Pkg: "dpkg"}, false), DeepEquals, []*Package{s.packages[13]})
+
+	// allMatches = True
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "mail-agent"}, true), Contains, []*Package{s.packages2[0], s.packages2[1]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "puppy"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionEqual, Version: "3"}, true), Contains, []*Package{s.packages2[5]})
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLess, Version: "1.2"}, true), Contains, []*Package{s.packages2[2], s.packages2[3]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLess, Version: "1.1~"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.2"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.1-bp1"}, true), Contains, []*Package{s.packages2[2]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionLessOrEqual, Version: "1.0"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreater, Version: "1.1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreater, Version: "5.0"}, true), IsNil)
+
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}, true), Contains, []*Package{s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
+	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "5.0"}, true), IsNil)
+}
+
+func (s *PackageListSuite) TestFilter(c *C) {
+	c.Check(func() { s.list.Filter([]PackageQuery{&PkgQuery{"abcd", "0.3", "i386"}}, false, nil, 0, nil) }, Panics, "list not indexed, can't filter")
+
+	plString := func(l *PackageList) string {
+		list := make([]string, 0, l.Len())
+		for _, p := range l.packages {
+			list = append(list, p.String())
+		}
+
+		sort.Strings(list)
+
+		return strings.Join(list, " ")
+	}
+
+	result, err := s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, &PkgQuery{"dpkg", "1.7", "source"},
+		&PkgQuery{"dpkg", "1.8", "amd64"}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_source")
+
+	result, err = s.il.Filter([]PackageQuery{
+		&DependencyQuery{Dep: Dependency{Pkg: "app"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "xyz"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "aa", Relation: VersionGreater, Version: "3.0"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source")
+
+	result, err = s.il.Filter([]PackageQuery{&DependencyQuery{Dep: Dependency{Pkg: "app", Architecture: "i386"}}}, true, NewPackageList(), 0, []string{"i386"})
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386")
+
+	result, err = s.il.Filter([]PackageQuery{
+		&DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "0.9"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "lib"}},
+		&DependencyQuery{Dep: Dependency{Pkg: "data"}}}, true, NewPackageList(), 0, []string{"i386", "amd64"})
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "")
+
+	result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"},
+		&FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_i386 data_1.1~bp1_all")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{&FieldQuery{Field: "Version", Relation: VersionGreaterOrEqual, Value: "1.0"},
+		&FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 data_1.1~bp1_all")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&NotQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.6.1-3_arm dpkg_1.6.1-3_source dpkg_1.7_source libx_1.5_arm")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "$Architecture", Relation: VersionRegexp, Value: "i.*6", Regexp: regexp.MustCompile("i.*6")}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
+}
+
+func (s *PackageListSuite) TestVerifyDependencies(c *C) {
+	missing, err := s.il.VerifyDependencies(0, []string{"i386"}, s.il, nil)
+	c.Check(err, IsNil)
+	c.Check(missing, DeepEquals, []Dependency{})
+
+	missing, err = s.il.VerifyDependencies(0, []string{"i386", "amd64"}, s.il, nil)
+	c.Check(err, IsNil)
+	c.Check(missing, DeepEquals, []Dependency{{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "amd64"}})
+
+	missing, err = s.il.VerifyDependencies(0, []string{"arm"}, s.il, nil)
+	c.Check(err, IsNil)
+	c.Check(missing, DeepEquals, []Dependency{})
+
+	missing, err = s.il.VerifyDependencies(DepFollowAllVariants, []string{"arm"}, s.il, nil)
+	c.Check(err, IsNil)
+	c.Check(missing, DeepEquals, []Dependency{{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "arm"},
+		{Pkg: "mail-agent", Relation: VersionDontCare, Version: "", Architecture: "arm"}})
+
+	for _, p := range s.sourcePackages {
+		s.il.Add(p)
+	}
+
+	missing, err = s.il.VerifyDependencies(DepFollowSource, []string{"i386", "amd64"}, s.il, nil)
+	c.Check(err, IsNil)
+	c.Check(missing, DeepEquals, []Dependency{{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "amd64"}})
+
+	missing, err = s.il.VerifyDependencies(DepFollowSource, []string{"arm"}, s.il, nil)
+	c.Check(err, IsNil)
+	c.Check(missing, DeepEquals, []Dependency{{Pkg: "libx", Relation: VersionEqual, Version: "1.5", Architecture: "source"}})
+
+	_, err = s.il.VerifyDependencies(0, []string{"i386", "amd64", "s390"}, s.il, nil)
+	c.Check(err, ErrorMatches, "unable to process package app_1.0_s390:.*")
+}
+
+func (s *PackageListSuite) TestArchitectures(c *C) {
+	archs := s.il.Architectures(true)
+	sort.Strings(archs)
+	c.Check(archs, DeepEquals, []string{"amd64", "arm", "i386", "s390", "source"})
+
+	archs = s.il.Architectures(false)
+	sort.Strings(archs)
+	c.Check(archs, DeepEquals, []string{"amd64", "arm", "i386", "s390"})
+}

deb/local.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -7,16 +7,21 @@ import (
 	"github.com/smira/aptly/database"
 	"github.com/ugorji/go/codec"
 	"log"
+	"sync"
 )
 
 // LocalRepo is a collection of packages created locally
 type LocalRepo struct {
 	// Permanent internal ID
-	UUID string
+	UUID string `json:"-"`
 	// User-assigned name
 	Name string
 	// Comment
 	Comment string
+	// DefaultDistribution
+	DefaultDistribution string `codec:",omitempty"`
+	// DefaultComponent
+	DefaultComponent string `codec:",omitempty"`
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 }
@@ -84,6 +89,7 @@ func (repo *LocalRepo) RefKey() []byte {
 
 // LocalRepoCollection does listing, updating/adding/deleting of LocalRepos
 type LocalRepoCollection struct {
+	*sync.RWMutex
 	db   database.Storage
 	list []*LocalRepo
 }
@@ -91,7 +97,8 @@ type LocalRepoCollection struct {
 // NewLocalRepoCollection loads LocalRepos from DB and makes up collection
 func NewLocalRepoCollection(db database.Storage) *LocalRepoCollection {
 	result := &LocalRepoCollection{
-		db: db,
+		RWMutex: &sync.RWMutex{},
+		db:      db,
 	}
 
 	blobs := db.FetchByPrefix([]byte("L"))
@@ -100,7 +107,7 @@ func NewLocalRepoCollection(db database.Storage) *LocalRepoCollection {
 	for _, blob := range blobs {
 		r := &LocalRepo{}
 		if err := r.Decode(blob); err != nil {
-			log.Printf("Error decoding mirror: %s\n", err)
+			log.Printf("Error decoding repo: %s\n", err)
 		} else {
 			result.list = append(result.list, r)
 		}

deb/local_test.go

@@ -1,9 +1,10 @@
-package debian
+package deb
 
 import (
 	"errors"
 	"github.com/smira/aptly/database"
-	. "launchpad.net/gocheck"
+
+	. "gopkg.in/check.v1"
 )
 
 type LocalRepoSuite struct {

deb/package.go

@@ -1,6 +1,7 @@
-package debian
+package deb
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
@@ -24,8 +25,12 @@ type Package struct {
 	Provides []string
 	// Is this source package
 	IsSource bool
+	// Is this udeb package
+	IsUdeb bool
 	// Hash of files section
 	FilesHash uint64
+	// Is this >= 0.6 package?
+	V06Plus bool
 	// Offload fields
 	deps  *PackageDependencies
 	extra *Stanza
@@ -34,6 +39,11 @@ type Package struct {
 	collection *PackageCollection
 }
 
+// Check interface
+var (
+	_ json.Marshaler = &Package{}
+)
+
 // NewPackageFromControlFile creates Package from parsed Debian control file
 func NewPackageFromControlFile(input Stanza) *Package {
 	result := &Package{
@@ -41,6 +51,7 @@ func NewPackageFromControlFile(input Stanza) *Package {
 		Version:      input["Version"],
 		Architecture: input["Architecture"],
 		Source:       input["Source"],
+		V06Plus:      true,
 	}
 
 	delete(input, "Package")
@@ -50,12 +61,18 @@ func NewPackageFromControlFile(input Stanza) *Package {
 
 	filesize, _ := strconv.ParseInt(input["Size"], 10, 64)
 
+	md5, ok := input["MD5sum"]
+	if !ok {
+		// there are some broken repos out there with MD5 in wrong field
+		md5 = input["MD5Sum"]
+	}
+
 	result.UpdateFiles(PackageFiles{PackageFile{
 		Filename:     filepath.Base(input["Filename"]),
 		downloadPath: filepath.Dir(input["Filename"]),
 		Checksums: utils.ChecksumInfo{
 			Size:   filesize,
-			MD5:    strings.TrimSpace(input["MD5sum"]),
+			MD5:    strings.TrimSpace(md5),
 			SHA1:   strings.TrimSpace(input["SHA1"]),
 			SHA256: strings.TrimSpace(input["SHA256"]),
 		},
@@ -63,6 +80,7 @@ func NewPackageFromControlFile(input Stanza) *Package {
 
 	delete(input, "Filename")
 	delete(input, "MD5sum")
+	delete(input, "MD5Sum")
 	delete(input, "SHA1")
 	delete(input, "SHA256")
 	delete(input, "Size")
@@ -89,6 +107,7 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 		Version:            input["Version"],
 		Architecture:       "source",
 		SourceArchitecture: input["Architecture"],
+		V06Plus:            true,
 	}
 
 	delete(input, "Package")
@@ -165,9 +184,26 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 	return result, nil
 }
 
+// NewUdebPackageFromControlFile creates .udeb Package from parsed Debian control file
+func NewUdebPackageFromControlFile(input Stanza) *Package {
+	p := NewPackageFromControlFile(input)
+	p.IsUdeb = true
+
+	return p
+}
+
 // Key returns unique key identifying package
 func (p *Package) Key(prefix string) []byte {
-	return []byte(prefix + "P" + p.Architecture + " " + p.Name + " " + p.Version)
+	if p.V06Plus {
+		return []byte(fmt.Sprintf("%sP%s %s %s %08x", prefix, p.Architecture, p.Name, p.Version, p.FilesHash))
+	}
+
+	return p.ShortKey(prefix)
+}
+
+// ShortKey returns key for the package that should be unique in one list
+func (p *Package) ShortKey(prefix string) []byte {
+	return []byte(fmt.Sprintf("%sP%s %s %s", prefix, p.Architecture, p.Name, p.Version))
 }
 
 // String creates readable representation
@@ -175,6 +211,82 @@ func (p *Package) String() string {
 	return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.Architecture)
 }
 
+// MarshalJSON implements json.Marshaller interface
+func (p *Package) MarshalJSON() ([]byte, error) {
+	stanza := p.Stanza()
+	stanza["FilesHash"] = fmt.Sprintf("%08x", p.FilesHash)
+	stanza["Key"] = string(p.Key(""))
+	stanza["ShortKey"] = string(p.ShortKey(""))
+
+	return json.Marshal(stanza)
+}
+
+// GetField returns fields from package
+func (p *Package) GetField(name string) string {
+	switch name {
+	// $Version is handled in FieldQuery
+	case "$Source":
+		if p.IsSource {
+			return ""
+		}
+		source := p.Source
+		if source == "" {
+			return p.Name
+		} else if pos := strings.Index(source, "("); pos != -1 {
+			return strings.TrimSpace(source[:pos])
+		}
+		return source
+	case "$SourceVersion":
+		if p.IsSource {
+			return ""
+		}
+		source := p.Source
+		if pos := strings.Index(source, "("); pos != -1 {
+			if pos2 := strings.LastIndex(source, ")"); pos2 != -1 && pos2 > pos {
+				return strings.TrimSpace(source[pos+1 : pos2])
+			}
+		}
+		return p.Version
+	case "$Architecture":
+		return p.Architecture
+	case "$PackageType":
+		if p.IsSource {
+			return "source"
+		}
+		if p.IsUdeb {
+			return "udeb"
+		}
+		return "deb"
+	case "Name":
+		return p.Name
+	case "Version":
+		return p.Version
+	case "Architecture":
+		if p.IsSource {
+			return p.SourceArchitecture
+		}
+		return p.Architecture
+	case "Source":
+		return p.Source
+	case "Depends":
+		return strings.Join(p.Deps().Depends, ", ")
+	case "Pre-Depends":
+		return strings.Join(p.Deps().PreDepends, ", ")
+	case "Suggests":
+		return strings.Join(p.Deps().Suggests, ", ")
+	case "Recommends":
+		return strings.Join(p.Deps().Recommends, ", ")
+	case "Provides":
+		return strings.Join(p.Provides, ", ")
+	case "Build-Depends":
+		return strings.Join(p.Deps().BuildDepends, ", ")
+	case "Build-Depends-Indep":
+		return strings.Join(p.Deps().BuildDependsInDep, ", ")
+	default:
+		return p.Extra()[name]
+	}
+}
+
 // MatchesArchitecture checks whether packages matches specified architecture
 func (p *Package) MatchesArchitecture(arch string) bool {
 	if p.Architecture == "all" && arch != "source" {
@@ -186,19 +298,23 @@ func (p *Package) MatchesArchitecture(arch string) bool {
 
 // MatchesDependency checks whether package matches specified dependency
 func (p *Package) MatchesDependency(dep Dependency) bool {
-	if dep.Pkg != p.Name {
-		return false
-	}
-
 	if dep.Architecture != "" && !p.MatchesArchitecture(dep.Architecture) {
 		return false
 	}
 
 	if dep.Relation == VersionDontCare {
-		return true
+		if utils.StrSliceHasItem(p.Provides, dep.Pkg) {
+			return true
+		}
+		return dep.Pkg == p.Name
+	}
+
+	if dep.Pkg != p.Name {
+		return false
 	}
 
 	r := CompareVersions(p.Version, dep.Version)
+
 	switch dep.Relation {
 	case VersionEqual:
 		return r == 0
@@ -210,6 +326,11 @@ func (p *Package) MatchesDependency(dep Dependency) bool {
 		return r <= 0
 	case VersionGreaterOrEqual:
 		return r >= 0
+	case VersionPatternMatch:
+		matched, err := filepath.Match(dep.Version, p.Version)
+		return err == nil && matched
+	case VersionRegexp:
+		return dep.Regexp.FindStringIndex(p.Version) != nil
 	}
 
 	panic("unknown relation")
@@ -305,7 +426,9 @@ func (p *Package) Stanza() (result Stanza) {
 		result["Architecture"] = p.SourceArchitecture
 	} else {
 		result["Architecture"] = p.Architecture
-		result["Source"] = p.Source
+		if p.Source != "" {
+			result["Source"] = p.Source
+		}
 	}
 
 	if p.IsSource {
@@ -376,7 +499,8 @@ func (p *Package) Equals(p2 *Package) bool {
 }
 
 // LinkFromPool links package file from pool to dist's pool location
-func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool, prefix string, component string) error {
+func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool,
+	prefix, component string, force bool) error {
 	poolDir, err := p.PoolDirectory()
 	if err != nil {
 		return err
@@ -388,16 +512,18 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 			return err
 		}
 
-		relPath, err := publishedStorage.LinkFromPool(prefix, component, poolDir, packagePool, sourcePath)
+		relPath := filepath.Join("pool", component, poolDir)
+		publishedDirectory := filepath.Join(prefix, relPath)
+
+		err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePath, f.Checksums.MD5, force)
 		if err != nil {
 			return err
 		}
 
-		dir := filepath.Dir(relPath)
 		if p.IsSource {
-			p.Extra()["Directory"] = dir
+			p.Extra()["Directory"] = relPath
 		} else {
-			p.Files()[i].downloadPath = dir
+			p.Files()[i].downloadPath = relPath
 		}
 	}
 
@@ -409,6 +535,8 @@ func (p *Package) PoolDirectory() (string, error) {
 	source := p.Source
 	if source == "" {
 		source = p.Name
+	} else if pos := strings.Index(source, "("); pos != -1 {
+		source = strings.TrimSpace(source[:pos])
 	}
 
 	if len(source) < 2 {

deb/package_collection.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"bytes"
@@ -12,12 +12,19 @@ import (
 type PackageCollection struct {
 	db           database.Storage
 	encodeBuffer bytes.Buffer
+	codecHandle  *codec.MsgpackHandle
 }
 
+// Verify interface
+var (
+	_ PackageCatalog = &PackageCollection{}
+)
+
 // NewPackageCollection creates new PackageCollection and binds it to database
 func NewPackageCollection(db database.Storage) *PackageCollection {
 	return &PackageCollection{
-		db: db,
+		db:          db,
+		codecHandle: &codec.MsgpackHandle{},
 	}
 }
 
@@ -53,7 +60,7 @@ func (collection *PackageCollection) ByKey(key []byte) (*Package, error) {
 	if len(encoded) > 2 && (encoded[0] != 0xc1 || encoded[1] != 0x1) {
 		oldp := &oldPackage{}
 
-		decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+		decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 		err = decoder.Decode(oldp)
 		if err != nil {
 			return nil, err
@@ -83,12 +90,12 @@ func (collection *PackageCollection) ByKey(key []byte) (*Package, error) {
 		p.UpdateFiles(PackageFiles(oldp.Files))
 
 		// Save in new format
-		err = collection.internalUpdate(p)
+		err = collection.Update(p)
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		decoder := codec.NewDecoderBytes(encoded[2:], &codec.MsgpackHandle{})
+		decoder := codec.NewDecoderBytes(encoded[2:], collection.codecHandle)
 		err = decoder.Decode(p)
 		if err != nil {
 			return nil, err
@@ -109,7 +116,7 @@ func (collection *PackageCollection) loadExtra(p *Package) *Stanza {
 
 	stanza := &Stanza{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(stanza)
 	if err != nil {
 		panic("unable to decode extra")
@@ -127,7 +134,7 @@ func (collection *PackageCollection) loadDependencies(p *Package) *PackageDepend
 
 	deps := &PackageDependencies{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(deps)
 	if err != nil {
 		panic("unable to decode deps")
@@ -145,7 +152,7 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 
 	files := &PackageFiles{}
 
-	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
 	err = decoder.Decode(files)
 	if err != nil {
 		panic("unable to decode files")
@@ -156,26 +163,7 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 
 // Update adds or updates information about package in DB checking for conficts first
 func (collection *PackageCollection) Update(p *Package) error {
-	existing, err := collection.ByKey(p.Key(""))
+	encoder := codec.NewEncoder(&collection.encodeBuffer, collection.codecHandle)
-	if err == nil {
-		// if .Files is different, consider to be conflict
-		if p.FilesHash != existing.FilesHash {
-			return fmt.Errorf("unable to save: %s, conflict with existing packge", p)
-		}
-		// ok, .Files are the same, but maybe some meta-data is different, proceed to saving
-	} else {
-		if err != database.ErrNotFound {
-			return err
-		}
-		// ok, package doesn't exist yet
-	}
-
-	return collection.internalUpdate(p)
-}
-
-// internalUpdate updates information in DB about package and offloaded fields
-func (collection *PackageCollection) internalUpdate(p *Package) error {
-	encoder := codec.NewEncoder(&collection.encodeBuffer, &codec.MsgpackHandle{})
 
 	collection.encodeBuffer.Reset()
 	collection.encodeBuffer.WriteByte(0xc1)
@@ -254,3 +242,49 @@ func (collection *PackageCollection) DeleteByKey(key []byte) error {
 	}
 	return nil
 }
+
+// Scan does full scan on all the packages
+func (collection *PackageCollection) Scan(q PackageQuery) (result *PackageList) {
+	result = NewPackageList()
+
+	for _, key := range collection.db.KeysByPrefix([]byte("P")) {
+		pkg, err := collection.ByKey(key)
+		if err != nil {
+			panic(fmt.Sprintf("unable to load package: %s", err))
+		}
+
+		if q.Matches(pkg) {
+			result.Add(pkg)
+		}
+	}
+
+	return
+}
+
+// Search is not implemented
+func (collection *PackageCollection) Search(dep Dependency, allMatches bool) (searchResults []*Package) {
+	panic("Not implemented")
+}
+
+// SearchSupported returns false
+func (collection *PackageCollection) SearchSupported() bool {
+	return false
+}
+
+// SearchByKey finds package by exact key
+func (collection *PackageCollection) SearchByKey(arch, name, version string) (result *PackageList) {
+	result = NewPackageList()
+
+	for _, key := range collection.db.KeysByPrefix([]byte(fmt.Sprintf("P%s %s %s", arch, name, version))) {
+		pkg, err := collection.ByKey(key)
+		if err != nil {
+			panic(fmt.Sprintf("unable to load package: %s", err))
+		}
+
+		if pkg.Architecture == arch && pkg.Name == name && pkg.Version == version {
+			result.Add(pkg)
+		}
+	}
+
+	return
+}

deb/package_collection_test.go

@@ -1,9 +1,10 @@
-package debian
+package deb
 
 import (
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/utils"
-	. "launchpad.net/gocheck"
+
+	. "gopkg.in/check.v1"
 )
 
 type PackageCollectionSuite struct {
@@ -48,20 +49,6 @@ func (s *PackageCollectionSuite) TestUpdate(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(res.Equals(s.p), Equals, false)
 	c.Assert(res.Equals(p2), Equals, true)
-
-	// change file info
-	p2 = NewPackageFromControlFile(packageStanza.Copy())
-	p2.UpdateFiles(nil)
-	res, err = s.collection.ByKey(p2.Key(""))
-	err = s.collection.Update(p2)
-	c.Assert(err, ErrorMatches, ".*conflict with existing packge")
-	p2 = NewPackageFromControlFile(packageStanza.Copy())
-	files := p2.Files()
-	files[0].Checksums.MD5 = "abcdef"
-	p2.UpdateFiles(files)
-	res, err = s.collection.ByKey(p2.Key(""))
-	err = s.collection.Update(p2)
-	c.Assert(err, ErrorMatches, ".*conflict with existing packge")
 }
 
 func (s *PackageCollectionSuite) TestByKey(c *C) {

deb/package_deps.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"strings"

deb/package_files.go

@@ -1,4 +1,4 @@
-package debian
+package deb
 
 import (
 	"encoding/binary"