Version bump to 0.9.1

full reference list collection. #217

Fixes bug with aptly db cleanup removing conflicting packages.

.travis.yml

@@ -1,17 +1,20 @@
 language: go
 
 go:
-  - 1.2.1
+  - 1.3.3
-  - 1.3.1
   - tip
 
 env:
   global:
     - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
     - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
+    - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
 before_install:
   - sudo apt-get update -qq
-  - sudo apt-get install -y python-boto
+  - sudo apt-get install -y python-virtualenv graphviz
+  - virtualenv env
+  - . env/bin/activate
+  - pip install boto requests python-swiftclient
 install:
   - make prepare
 
@@ -21,3 +24,12 @@ script: make travis
 matrix:
     allow_failures:
         - go: tip
+
+
+notifications:
+  webhooks:
+    urls:
+      - "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
+    on_success: change  # options: [always|never|change] default: always
+    on_failure: always  # options: [always|never|change] default: always
+    on_start: false     # default: false

AUTHORS

@@ -3,5 +3,15 @@ List of contributors, in chronological order:
 * Andrey Smirnov (https://github.com/smira)
 * Sebastien Binet (https://github.com/sbinet)
 * Ryan Uber (https://github.com/ryanuber)
-* Simon Aquino (https://github.com/simonaquino)
+* Simon Aquino (https://github.com/queeno)
-* Vincent Batoufflet (https://github.com/vbatoufflet)
+* Vincent Batoufflet (https://github.com/vbatoufflet)
+* Ivan Kurnosov (https://github.com/zerkms)
+* Dmitrii Kashin (https://github.com/freehck)
+* Chris Read (https://github.com/cread)
+* Rohan Garg (https://github.com/shadeslayer)
+* Russ Allbery (https://github.com/rra)
+* Sylvain Baubeau (https://github.com/lebauce)
+* Andrea Bernardo Ciddio (https://github.com/bcandrea)
+* Michael Koval (https://github.com/mkoval)
+* Alexander Guy (https://github.com/alexanderguy)
+* Sebastien Badia (https://github.com/sbadia)

Gomfile

@@ -1,27 +1,33 @@
 gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
-gom 'code.google.com/p/go.crypto/ssh/terminal', :commit => '7aa593ce8cea'
 gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2'
 gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556'
 gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
-gom 'github.com/cheggaaa/pb', :commit => '74be7a1388046f374ac36e93d46f5d56e856f827'
+gom 'github.com/AlekSi/pointer', :commit => '5f6d527dae3d678b46fbb20331ddf44e2b841943'
-gom 'github.com/vaughan0/go-ini', :commit => 'a98ad7ee00ec53921f08832bc06ecf7fd600e6a1'
+gom 'github.com/cheggaaa/pb', :commit => '2c1b74620cc58a81ac152ee2d322e28c806d81ed'
+gom 'github.com/gin-gonic/gin', :commit => 'b1758d3bfa09e61ddbc1c9a627e936eec6a170de'
+gom 'github.com/jlaffaye/ftp', :commit => 'fec71e62e457557fbe85cefc847a048d57815d76'
+gom 'github.com/julienschmidt/httprouter', :commit => '46807412fe50aaceb73bb57061c2230fd26a1640'
 gom 'github.com/mattn/go-shellwords', :commit => 'c7ca6f94add751566a61cf2199e1de78d4c3eee4'
 gom 'github.com/mitchellh/goamz/s3', :commit => 'e7664b32019f31fd1bdf33f9e85f28722f700405'
 gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
+gom 'github.com/ncw/swift', :commit => '384ef27c70645e285f8bb9d02276bf654d06027e'
 gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
 gom 'github.com/smira/flag', :commit => '357ed3e599ffcbd4aeaa828e1d10da2df3ea5107'
 gom 'github.com/smira/go-ftp-protocol/protocol', :commit => '066b75c2b70dca7ae10b1b88b47534a3c31ccfaa'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => 'e2fa4e6ac1cc41a73bc9fd467878ecbf65df5cc3'
+gom 'github.com/syndtr/gosnappy/snappy', :commit => 'ce8acff4829e0c2458a67ead32390ac0a381c862'
+gom 'github.com/syndtr/goleveldb/leveldb', :commit => '97e257099d2ab9578151ba85e2641e2cd14d3ca8'
 gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
+gom 'github.com/vaughan0/go-ini', :commit => 'a98ad7ee00ec53921f08832bc06ecf7fd600e6a1'
 gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
+gom 'golang.org/x/crypto/ssh/terminal', :commit => 'a7ead6ddf06233883deca151dffaef2effbf498f'
 
 group :test do
-    gom 'launchpad.net/gocheck'
+    gom 'gopkg.in/check.v1'
 end
 
 group :development do
    gom 'github.com/golang/lint/golint'
    gom 'github.com/mattn/goveralls'
    gom 'github.com/axw/gocov/gocov'
-   gom 'code.google.com/p/go.tools/cmd/cover'
+   gom 'golang.org/x/tools/cmd/cover'
 end

Makefile

@@ -1,6 +1,6 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-PACKAGES=database deb files http query s3 utils
+PACKAGES=context database deb files http query swift s3 utils
-ALL_PACKAGES=aptly cmd console database deb files http query s3 utils
+ALL_PACKAGES=api aptly context cmd console database deb files http query swift s3 utils
 BINPATH=$(abspath ./_vendor/bin)
 GOM_ENVIRONMENT=-test
 PYTHON?=python
@@ -36,7 +36,7 @@ coverage: coverage.out
 	rm -f coverage.out
 
 check:
-	$(GOM) exec go tool vet -all=true -shadow=true $(ALL_PACKAGES:%=./%)
+	$(GOM) exec go tool vet -all=true $(ALL_PACKAGES:%=./%)
 	$(GOM) exec golint $(ALL_PACKAGES:%=./%)
 
 install:
@@ -67,7 +67,7 @@ package:
 	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
 	gzip root/usr/share/man/man1/aptly.1
 	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
-	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 -C root/ .
+	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 --deb-recommends graphviz -C root/ .
 	mv aptly_$(VERSION)_*.deb ~
 
 src-package:
@@ -77,6 +77,8 @@ src-package:
 	cd aptly-$(VERSION)/src/github.com/smira/aptly && gom -production install
 	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . \( -name .git -o -name .bzr -o -name .hg \) -print | xargs rm -rf
 	rm -rf aptly-$(VERSION)/src/github.com/smira/aptly/_vendor/{pkg,bin}
+	mkdir -p aptly-$(VERSION)/bash_completion.d
+	(cd aptly-$(VERSION)/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/$(VERSION)/aptly)
 	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
 	rm -rf aptly-$(VERSION)
 	curl -T aptly-$(VERSION)-src.tar.bz2 -usmira:$(BINTRAY_KEY) https://api.bintray.com/content/smira/aptly/aptly/$(VERSION)/$(VERSION)/aptly-$(VERSION)-src.tar.bz2

README.rst

@@ -8,8 +8,11 @@ aptly
 .. image:: https://coveralls.io/repos/smira/aptly/badge.png?branch=HEAD
     :target: https://coveralls.io/r/smira/aptly?branch=HEAD
 
-.. image:: http://gobuild.io/badge/github.com/smira/aptly/download.png
+.. image:: https://badges.gitter.im/Join Chat.svg
-    :target: http://gobuild.io/github.com/smira/aptly
+    :target: https://gitter.im/smira/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+
+.. image:: http://goreportcard.com/badge/gojp/goreportcard
+    :target: http://goreportcard.com/report/gojp/goreportcard
 
 Aptly is a swiss army knife for Debian repository management.
 
@@ -28,6 +31,7 @@ Aptly features: ("+" means planned features)
 * merge two or more snapshots into one
 * filter repository by search query, pulling dependencies when required
 * publish self-made packages as Debian repositories
+* REST API for remote access
 * mirror repositories "as-is" (without resigning with user's key) (+)
 * support for yum repositories (+)
 
@@ -44,8 +48,7 @@ To install aptly on Debian/Ubuntu, add new repository to /etc/apt/sources.list::
 
 And import key that is used to sign the release::
 
-    $ gpg --keyserver keys.gnupg.net --recv-keys 2A194991
+    $ apt-key adv --keyserver keys.gnupg.net --recv-keys E083A3782A194991
-    $ gpg -a --export 2A194991 | sudo apt-key add -
 
 After that you can install aptly as any other software package::
 
@@ -55,9 +58,13 @@ After that you can install aptly as any other software package::
 Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
 Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
 
+If you would like to use nightly builds (unstable), please use following repository::
+
+    deb http://repo.aptly.info/ nightly main
+
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.2+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.3+ required)::
 
     go get -u github.com/mattn/gom
     mkdir -p $GOPATH/src/github.com/smira/aptly

api/api.go

@@ -0,0 +1,113 @@
+// Package api provides implementation of aptly REST API
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"sort"
+	"time"
+)
+
+// Lock order acquisition (canonical):
+//  1. RemoteRepoCollection
+//  2. LocalRepoCollection
+//  3. SnapshotCollection
+//  4. PublishedRepoCollection
+
+// GET /api/version
+func apiVersion(c *gin.Context) {
+	c.JSON(200, gin.H{"Version": aptly.Version})
+}
+
+// Periodically flushes CollectionFactory to free up memory used by collections,
+// flushing caches.
+//
+// Should be run in goroutine!
+func cacheFlusher() {
+	ticker := time.Tick(15 * time.Minute)
+
+	for {
+		<-ticker
+
+		// lock everything to eliminate in-progress calls
+		r := context.CollectionFactory().RemoteRepoCollection()
+		r.Lock()
+		defer r.Unlock()
+
+		l := context.CollectionFactory().LocalRepoCollection()
+		l.Lock()
+		defer l.Unlock()
+
+		s := context.CollectionFactory().SnapshotCollection()
+		s.Lock()
+		defer s.Unlock()
+
+		p := context.CollectionFactory().PublishedRepoCollection()
+		p.Lock()
+		defer p.Unlock()
+
+		// all collections locked, flush them
+		context.CollectionFactory().Flush()
+	}
+}
+
+// Common piece of code to show list of packages,
+// with searching & details if requested
+func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
+	result := []*deb.Package{}
+
+	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), nil)
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	queryS := c.Request.URL.Query().Get("q")
+	if queryS != "" {
+		q, err := query.Parse(c.Request.URL.Query().Get("q"))
+		if err != nil {
+			c.Fail(400, err)
+			return
+		}
+
+		withDeps := c.Request.URL.Query().Get("withDeps") == "1"
+		architecturesList := []string{}
+
+		if withDeps {
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = list.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				return
+			}
+		}
+
+		list.PrepareIndex()
+
+		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
+			nil, context.DependencyOptions(), architecturesList)
+		if err != nil {
+			c.Fail(500, fmt.Errorf("unable to search: %s", err))
+		}
+	}
+
+	if c.Request.URL.Query().Get("format") == "details" {
+		list.ForEach(func(p *deb.Package) error {
+			result = append(result, p)
+			return nil
+		})
+
+		c.JSON(200, result)
+	} else {
+		c.JSON(200, list.Strings())
+	}
+}

api/files.go

@@ -0,0 +1,185 @@
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func verifyPath(path string) bool {
+	path = filepath.Clean(path)
+	for _, part := range strings.Split(path, string(filepath.Separator)) {
+		if part == ".." || part == "." {
+			return false
+		}
+	}
+
+	return true
+
+}
+
+func verifyDir(c *gin.Context) bool {
+	if !verifyPath(c.Params.ByName("dir")) {
+		c.Fail(400, fmt.Errorf("wrong dir"))
+		return false
+	}
+
+	return true
+}
+
+// GET /files
+func apiFilesListDirs(c *gin.Context) {
+	list := []string{}
+
+	err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == context.UploadPath() {
+			return nil
+		}
+
+		if info.IsDir() {
+			list = append(list, filepath.Base(path))
+			return filepath.SkipDir
+		}
+
+		return nil
+	})
+
+	if err != nil && !os.IsNotExist(err) {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// POST /files/:dir/
+func apiFilesUpload(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	err := os.MkdirAll(path, 0777)
+
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	stored := []string{}
+
+	for _, files := range c.Request.MultipartForm.File {
+		for _, file := range files {
+			src, err := file.Open()
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+			defer src.Close()
+
+			destPath := filepath.Join(path, filepath.Base(file.Filename))
+			dst, err := os.Create(destPath)
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+			defer dst.Close()
+
+			_, err = io.Copy(dst, src)
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+
+			stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
+		}
+	}
+
+	c.JSON(200, stored)
+
+}
+
+// GET /files/:dir
+func apiFilesListFiles(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	list := []string{}
+	root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == root {
+			return nil
+		}
+
+		list = append(list, filepath.Base(path))
+
+		return nil
+	})
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			c.Fail(404, err)
+		} else {
+			c.Fail(500, err)
+		}
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// DELETE /files/:dir
+func apiFilesDeleteDir(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// DELETE /files/:dir/:name
+func apiFilesDeleteFile(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	if !verifyPath(c.Params.ByName("name")) {
+		c.Fail(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
+	if err != nil {
+		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
+			c.Fail(500, err)
+			return
+		}
+	}
+
+	c.JSON(200, gin.H{})
+}

api/graph.go

@@ -0,0 +1,75 @@
+package api
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/deb"
+	"io"
+	"mime"
+	"os"
+	"os/exec"
+)
+
+// GET /api/graph.:ext
+func apiGraph(c *gin.Context) {
+	var (
+		err    error
+		output []byte
+	)
+
+	ext := c.Params.ByName("ext")
+
+	factory := context.CollectionFactory()
+
+	factory.RemoteRepoCollection().RLock()
+	defer factory.RemoteRepoCollection().RUnlock()
+	factory.LocalRepoCollection().RLock()
+	defer factory.LocalRepoCollection().RUnlock()
+	factory.SnapshotCollection().RLock()
+	defer factory.SnapshotCollection().RUnlock()
+	factory.PublishedRepoCollection().RLock()
+	defer factory.PublishedRepoCollection().RUnlock()
+
+	graph, err := deb.BuildGraph(factory)
+	if err != nil {
+		c.JSON(500, err)
+		return
+	}
+
+	buf := bytes.NewBufferString(graph.String())
+
+	command := exec.Command("dot", "-T"+ext)
+	command.Stderr = os.Stderr
+
+	stdin, err := command.StdinPipe()
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	_, err = io.Copy(stdin, buf)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	err = stdin.Close()
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	output, err = command.Output()
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		return
+	}
+
+	mimeType := mime.TypeByExtension("." + ext)
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+
+	c.Data(200, mimeType, output)
+}

api/packages.go

@@ -0,0 +1,16 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/packages/:key
+func apiPackagesShow(c *gin.Context) {
+	p, err := context.CollectionFactory().PackageCollection().ByKey([]byte(c.Params.ByName("key")))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	c.JSON(200, p)
+}

api/publish.go

@@ -0,0 +1,336 @@
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
+	"strings"
+)
+
+// SigningOptions is a shared between publish API GPG options structure
+type SigningOptions struct {
+	Skip           bool
+	Batch          bool
+	GpgKey         string
+	Keyring        string
+	SecretKeyring  string
+	Passphrase     string
+	PassphraseFile string
+}
+
+func getSigner(options *SigningOptions) (utils.Signer, error) {
+	if options.Skip {
+		return nil, nil
+	}
+
+	signer := &utils.GpgSigner{}
+	signer.SetKey(options.GpgKey)
+	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
+	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
+	signer.SetBatch(options.Batch)
+
+	err := signer.Init()
+	if err != nil {
+		return nil, err
+	}
+
+	return signer, nil
+}
+
+// Replace '_' with '/' and double '__' with single '_'
+func parseEscapedPath(path string) string {
+	result := strings.Replace(strings.Replace(path, "_", "/", -1), "//", "_", -1)
+	if result == "" {
+		result = "."
+	}
+	return result
+}
+
+// GET /publish
+func apiPublishList(c *gin.Context) {
+	localCollection := context.CollectionFactory().LocalRepoCollection()
+	localCollection.RLock()
+	defer localCollection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.RLock()
+	defer snapshotCollection.RUnlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	result := make([]*deb.PublishedRepo, 0, collection.Len())
+
+	err := collection.ForEach(func(repo *deb.PublishedRepo) error {
+		err := collection.LoadComplete(repo, context.CollectionFactory())
+		if err != nil {
+			return err
+		}
+
+		result = append(result, repo)
+
+		return nil
+	})
+
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, result)
+}
+
+// POST /publish/:prefix
+func apiPublishRepoOrSnapshot(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+
+	var b struct {
+		SourceKind string `binding:"required"`
+		Sources    []struct {
+			Component string
+			Name      string `binding:"required"`
+		} `binding:"required"`
+		Distribution   string
+		Label          string
+		Origin         string
+		ForceOverwrite bool
+		Architectures  []string
+		Signing        SigningOptions
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	if len(b.Sources) == 0 {
+		c.Fail(400, fmt.Errorf("unable to publish: soures are empty"))
+		return
+	}
+
+	var components []string
+	var sources []interface{}
+
+	if b.SourceKind == "snapshot" {
+		var snapshot *deb.Snapshot
+
+		snapshotCollection := context.CollectionFactory().SnapshotCollection()
+		snapshotCollection.RLock()
+		defer snapshotCollection.RUnlock()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+
+			snapshot, err = snapshotCollection.ByName(source.Name)
+			if err != nil {
+				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			sources = append(sources, snapshot)
+		}
+	} else if b.SourceKind == "local" {
+		var localRepo *deb.LocalRepo
+
+		localCollection := context.CollectionFactory().LocalRepoCollection()
+		localCollection.RLock()
+		defer localCollection.RUnlock()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+
+			localRepo, err = localCollection.ByName(source.Name)
+			if err != nil {
+				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			err = localCollection.LoadComplete(localRepo)
+			if err != nil {
+				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+			}
+
+			sources = append(sources, localRepo)
+		}
+	} else {
+		c.Fail(400, fmt.Errorf("unknown SourceKind"))
+		return
+	}
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, context.CollectionFactory())
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		return
+	}
+	published.Origin = b.Origin
+	published.Label = b.Label
+
+	duplicate := collection.CheckDuplicate(published)
+	if duplicate != nil {
+		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
+		c.Fail(400, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate))
+		return
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		return
+	}
+
+	err = collection.Add(published)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+	}
+
+	c.JSON(201, published)
+}
+
+// PUT /publish/:prefix/:distribution
+func apiPublishUpdateSwitch(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	var b struct {
+		ForceOverwrite bool
+		Signing        SigningOptions
+		Snapshots      []struct {
+			Component string `binding:"required"`
+			Name      string `binding:"required"`
+		}
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	// published.LoadComplete would touch local repo collection
+	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
+	localRepoCollection.RLock()
+	defer localRepoCollection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.RLock()
+	defer snapshotCollection.RUnlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		c.Fail(404, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+	err = collection.LoadComplete(published, context.CollectionFactory())
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	var updatedComponents []string
+
+	if published.SourceKind == "local" {
+		if len(b.Snapshots) > 0 {
+			c.Fail(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
+			return
+		}
+		updatedComponents = published.Components()
+		for _, component := range updatedComponents {
+			published.UpdateLocalRepo(component)
+		}
+	} else if published.SourceKind == "snapshot" {
+		publishedComponents := published.Components()
+		for _, snapshotInfo := range b.Snapshots {
+			if !utils.StrSliceHasItem(publishedComponents, snapshotInfo.Component) {
+				c.Fail(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
+				return
+			}
+
+			snapshot, err := snapshotCollection.ByName(snapshotInfo.Name)
+			if err != nil {
+				c.Fail(404, err)
+				return
+			}
+
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+
+			published.UpdateSnapshot(snapshotInfo.Component, snapshot)
+			updatedComponents = append(updatedComponents, snapshotInfo.Component)
+		}
+	} else {
+		c.Fail(500, fmt.Errorf("unknown published repository type"))
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+	}
+
+	err = collection.Update(published)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+	}
+
+	err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
+		context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+	}
+
+	c.JSON(200, published)
+}
+
+// DELETE /publish/:prefix/:distribution
+func apiPublishDrop(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	// published.LoadComplete would touch local repo collection
+	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
+	localRepoCollection.RLock()
+	defer localRepoCollection.RUnlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	err := collection.Remove(context, storage, prefix, distribution,
+		context.CollectionFactory(), context.Progress())
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to drop: %s", err))
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}

api/repos.go

@@ -0,0 +1,370 @@
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
+	"os"
+	"path/filepath"
+)
+
+// GET /api/repos
+func apiReposList(c *gin.Context) {
+	result := []*deb.LocalRepo{}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	context.CollectionFactory().LocalRepoCollection().ForEach(func(r *deb.LocalRepo) error {
+		result = append(result, r)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/repos
+func apiReposCreate(c *gin.Context) {
+	var b struct {
+		Name                string `binding:"required"`
+		Comment             string
+		DefaultDistribution string
+		DefaultComponent    string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	repo := deb.NewLocalRepo(b.Name, b.Comment)
+	repo.DefaultComponent = b.DefaultComponent
+	repo.DefaultDistribution = b.DefaultDistribution
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	err := context.CollectionFactory().LocalRepoCollection().Add(repo)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, repo)
+}
+
+// PUT /api/repos/:name
+func apiReposEdit(c *gin.Context) {
+	var b struct {
+		Comment             string
+		DefaultDistribution string
+		DefaultComponent    string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	if b.Comment != "" {
+		repo.Comment = b.Comment
+	}
+	if b.DefaultDistribution != "" {
+		repo.DefaultDistribution = b.DefaultDistribution
+	}
+	if b.DefaultComponent != "" {
+		repo.DefaultComponent = b.DefaultComponent
+	}
+
+	err = collection.Update(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
+// GET /api/repos/:name
+func apiReposShow(c *gin.Context) {
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
+// DELETE /api/repos/:name
+func apiReposDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.RLock()
+	defer snapshotCollection.RUnlock()
+
+	publishedCollection := context.CollectionFactory().PublishedRepoCollection()
+	publishedCollection.RLock()
+	defer publishedCollection.RUnlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	published := publishedCollection.ByLocalRepo(repo)
+	if len(published) > 0 {
+		c.Fail(409, fmt.Errorf("unable to drop, local repo is published"))
+		return
+	}
+
+	if !force {
+		snapshots := snapshotCollection.ByLocalRepoSource(repo)
+		if len(snapshots) > 0 {
+			c.Fail(409, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override"))
+			return
+		}
+	}
+
+	err = collection.Drop(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// GET /api/repos/:name/packages
+func apiReposPackagesShow(c *gin.Context) {
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	showPackages(c, repo.RefList())
+}
+
+// Handler for both add and delete
+func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p *deb.Package) error) {
+	var b struct {
+		PackageRefs []string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	// verify package refs and build package list
+	for _, ref := range b.PackageRefs {
+		var p *deb.Package
+
+		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
+		if err != nil {
+			if err == database.ErrNotFound {
+				c.Fail(404, fmt.Errorf("package %s: %s", ref, err))
+			} else {
+				c.Fail(500, err)
+			}
+			return
+		}
+		err = cb(list, p)
+		if err != nil {
+			c.Fail(400, err)
+			return
+		}
+	}
+
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save: %s", err))
+		return
+	}
+
+	c.JSON(200, repo)
+
+}
+
+// POST /repos/:name/packages
+func apiReposPackagesAdd(c *gin.Context) {
+	apiReposPackagesAddDelete(c, func(list *deb.PackageList, p *deb.Package) error {
+		return list.Add(p)
+	})
+}
+
+// DELETE /repos/:name/packages
+func apiReposPackagesDelete(c *gin.Context) {
+	apiReposPackagesAddDelete(c, func(list *deb.PackageList, p *deb.Package) error {
+		list.Remove(p)
+		return nil
+	})
+}
+
+// POST /repos/:name/file/:dir/:file
+func apiReposPackageFromFile(c *gin.Context) {
+	// redirect all work to dir method
+	apiReposPackageFromDir(c)
+}
+
+// POST /repos/:name/file/:dir
+func apiReposPackageFromDir(c *gin.Context) {
+	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
+	noRemove := c.Request.URL.Query().Get("noRemove") == "1"
+
+	if !verifyDir(c) {
+		return
+	}
+
+	fileParam := c.Params.ByName("file")
+	if fileParam != "" && !verifyPath(fileParam) {
+		c.Fail(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	verifier := &utils.GpgVerifier{}
+
+	var (
+		sources                      []string
+		packageFiles, failedFiles    []string
+		processedFiles, failedFiles2 []string
+		reporter                     = &aptly.RecordingResultReporter{
+			Warnings:     []string{},
+			AddedLines:   []string{},
+			RemovedLines: []string{},
+		}
+		list *deb.PackageList
+	)
+
+	if fileParam == "" {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"))}
+	} else {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
+	}
+
+	packageFiles, failedFiles, err = deb.CollectPackageFiles(sources, reporter)
+
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to collect package files: %s", err))
+		return
+	}
+
+	list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to load packages: %s", err))
+		return
+	}
+
+	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+		context.CollectionFactory().PackageCollection(), reporter)
+	failedFiles = append(failedFiles, failedFiles2...)
+
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to import package files: %s", err))
+		return
+	}
+
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save: %s", err))
+		return
+	}
+
+	if !noRemove {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err := os.Remove(file)
+			if err != nil {
+				reporter.Warning("unable to remove file %s: %s", file, err)
+			}
+		}
+
+		// atempt to remove dir, if it fails, that's fine: probably it's not empty
+		os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	}
+
+	if failedFiles == nil {
+		failedFiles = []string{}
+	}
+
+	c.JSON(200, gin.H{
+		"Report":      reporter,
+		"FailedFiles": failedFiles,
+	})
+}

api/router.go

@@ -0,0 +1,82 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+	ctx "github.com/smira/aptly/context"
+	"net/http"
+)
+
+var context *ctx.AptlyContext
+
+// Router returns prebuilt with routes http.Handler
+func Router(c *ctx.AptlyContext) http.Handler {
+	context = c
+
+	go cacheFlusher()
+
+	router := gin.Default()
+	router.Use(gin.ErrorLogger())
+
+	root := router.Group("/api")
+
+	{
+		root.GET("/version", apiVersion)
+	}
+
+	{
+		root.GET("/repos", apiReposList)
+		root.POST("/repos", apiReposCreate)
+		root.GET("/repos/:name", apiReposShow)
+		root.PUT("/repos/:name", apiReposEdit)
+		root.DELETE("/repos/:name", apiReposDrop)
+
+		root.GET("/repos/:name/packages", apiReposPackagesShow)
+		root.POST("/repos/:name/packages", apiReposPackagesAdd)
+		root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
+
+		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
+		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+
+		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
+	}
+
+	{
+		root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
+	}
+
+	{
+		root.GET("/files", apiFilesListDirs)
+		root.POST("/files/:dir", apiFilesUpload)
+		root.GET("/files/:dir", apiFilesListFiles)
+		root.DELETE("/files/:dir", apiFilesDeleteDir)
+		root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
+	}
+
+	{
+		root.GET("/publish", apiPublishList)
+		root.POST("/publish", apiPublishRepoOrSnapshot)
+		root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
+		root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
+		root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+	}
+
+	{
+		root.GET("/snapshots", apiSnapshotsList)
+		root.POST("/snapshots", apiSnapshotsCreate)
+		root.PUT("/snapshots/:name", apiSnapshotsUpdate)
+		root.GET("/snapshots/:name", apiSnapshotsShow)
+		root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
+		root.DELETE("/snapshots/:name", apiSnapshotsDrop)
+		root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+	}
+
+	{
+		root.GET("/packages/:key", apiPackagesShow)
+	}
+
+	{
+		root.GET("/graph.:ext", apiGraph)
+	}
+
+	return router
+}

api/snapshot.go

@@ -0,0 +1,410 @@
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/deb"
+)
+
+// GET /api/snapshots
+func apiSnapshotsList(c *gin.Context) {
+	SortMethodString := c.Request.URL.Query().Get("sort")
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	if SortMethodString == "" {
+		SortMethodString = "name"
+	}
+
+	result := []*deb.Snapshot{}
+	collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
+		result = append(result, snapshot)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/mirrors/:name/snapshots/
+func apiSnapshotsCreateFromMirror(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.RemoteRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().RemoteRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	repo, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		c.Fail(409, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// POST /api/snapshots
+func apiSnapshotsCreate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name            string `binding:"required"`
+		Description     string
+		SourceSnapshots []string
+		PackageRefs     []string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	if b.Description == "" {
+		if len(b.SourceSnapshots)+len(b.PackageRefs) == 0 {
+			b.Description = "Created as empty"
+		}
+	}
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	sources := make([]*deb.Snapshot, len(b.SourceSnapshots))
+
+	for i := range b.SourceSnapshots {
+		sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
+		if err != nil {
+			c.Fail(404, err)
+			return
+		}
+
+		err = snapshotCollection.LoadComplete(sources[i])
+		if err != nil {
+			c.Fail(500, err)
+			return
+		}
+	}
+
+	list := deb.NewPackageList()
+
+	// verify package refs and build package list
+	for _, ref := range b.PackageRefs {
+		var p *deb.Package
+
+		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
+		if err != nil {
+			if err == database.ErrNotFound {
+				c.Fail(404, fmt.Errorf("package %s: %s", ref, err))
+			} else {
+				c.Fail(500, err)
+			}
+			return
+		}
+		err = list.Add(p)
+		if err != nil {
+			c.Fail(400, err)
+			return
+		}
+	}
+
+	snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// POST /api/repos/:name/snapshots
+func apiSnapshotsCreateFromRepository(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.LocalRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	repo, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// PUT /api/snapshots/:name
+func apiSnapshotsUpdate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string
+		Description string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshot, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	_, err = collection.ByName(b.Name)
+	if err == nil {
+		c.Fail(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
+		return
+	}
+
+	if b.Name != "" {
+		snapshot.Name = b.Name
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, snapshot)
+}
+
+// GET /api/snapshots/:name
+func apiSnapshotsShow(c *gin.Context) {
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, snapshot)
+}
+
+// DELETE /api/snapshots/:name
+func apiSnapshotsDrop(c *gin.Context) {
+	name := c.Params.ByName("name")
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	publishedCollection := context.CollectionFactory().PublishedRepoCollection()
+	publishedCollection.RLock()
+	defer publishedCollection.RUnlock()
+
+	snapshot, err := snapshotCollection.ByName(name)
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	published := publishedCollection.BySnapshot(snapshot)
+
+	if len(published) > 0 {
+		c.Fail(409, fmt.Errorf("unable to drop: snapshot is published"))
+		return
+	}
+
+	if !force {
+		snapshots := snapshotCollection.BySnapshotSource(snapshot)
+		if len(snapshots) > 0 {
+			c.Fail(409, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override"))
+			return
+		}
+	}
+
+	err = snapshotCollection.Drop(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// GET /api/snapshots/:name/diff/:withSnapshot
+func apiSnapshotsDiff(c *gin.Context) {
+	onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1"
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshotA, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotA)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotB)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	// Calculate diff
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	result := []deb.PackageDiff{}
+
+	for _, pdiff := range diff {
+		if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) {
+			continue
+		}
+
+		result = append(result, pdiff)
+	}
+
+	c.JSON(200, result)
+}
+
+// GET /api/snapshots/:name/packages
+func apiSnapshotsSearchPackages(c *gin.Context) {
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	showPackages(c, snapshot.RefList())
+}

aptly/report.go

@@ -0,0 +1,67 @@
+package aptly
+
+import (
+	"fmt"
+)
+
+// ResultReporter is abstraction for result reporting from complex processing functions
+type ResultReporter interface {
+	// Warning is non-fatal error message
+	Warning(msg string, a ...interface{})
+	// Removed is signal that something has been removed
+	Removed(msg string, a ...interface{})
+	// Added is signal that something has been added
+	Added(msg string, a ...interface{})
+}
+
+// ConsoleResultReporter is implementation of ResultReporter that prints in colors to console
+type ConsoleResultReporter struct {
+	Progress Progress
+}
+
+// Check interface
+var (
+	_ ResultReporter = &ConsoleResultReporter{}
+)
+
+// Warning is non-fatal error message (yellow)
+func (c *ConsoleResultReporter) Warning(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@y[!]@| @!"+msg+"@|", a...)
+}
+
+// Removed is signal that something has been removed (red)
+func (c *ConsoleResultReporter) Removed(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@r[-]@| "+msg, a...)
+}
+
+// Added is signal that something has been added (green)
+func (c *ConsoleResultReporter) Added(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@g[+]@| "+msg, a...)
+}
+
+// RecordingResultReporter is implementation of ResultReporter that collects all messages
+type RecordingResultReporter struct {
+	Warnings     []string
+	AddedLines   []string `json:"Added"`
+	RemovedLines []string `json:"Removed"`
+}
+
+// Check interface
+var (
+	_ ResultReporter = &RecordingResultReporter{}
+)
+
+// Warning is non-fatal error message
+func (r *RecordingResultReporter) Warning(msg string, a ...interface{}) {
+	r.Warnings = append(r.Warnings, fmt.Sprintf(msg, a...))
+}
+
+// Removed is signal that something has been removed
+func (r *RecordingResultReporter) Removed(msg string, a ...interface{}) {
+	r.RemovedLines = append(r.RemovedLines, fmt.Sprintf(msg, a...))
+}
+
+// Added is signal that something has been added
+func (r *RecordingResultReporter) Added(msg string, a ...interface{}) {
+	r.AddedLines = append(r.AddedLines, fmt.Sprintf(msg, a...))
+}

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
 // Version of aptly
-const Version = "0.8"
+const Version = "0.9.1"
 
 // Enable debugging features?
 const EnableDebug = false

cmd/api.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdAPI() *commander.Command {
+	return &commander.Command{
+		UsageLine: "api",
+		Short:     "start API server/issue requests",
+		Subcommands: []*commander.Command{
+			makeCmdAPIServe(),
+		},
+	}
+}

cmd/api_serve.go

@@ -0,0 +1,52 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/api"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"net/http"
+)
+
+func aptlyAPIServe(cmd *commander.Command, args []string) error {
+	var (
+		err error
+	)
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	listen := context.Flags().Lookup("listen").Value.String()
+
+	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
+
+	err = http.ListenAndServe(listen, api.Router(context))
+	if err != nil {
+		return fmt.Errorf("unable to serve: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdAPIServe() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyAPIServe,
+		UsageLine: "serve",
+		Short:     "start API HTTP service",
+		Long: `
+Stat HTTP server with aptly REST API.
+
+Example:
+
+  $ aptly api serve -listen=:8080
+`,
+		Flag: *flag.NewFlagSet("aptly-serve", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening")
+
+	return cmd
+
+}

cmd/cmd.go

@@ -65,17 +65,18 @@ fine-grained changes in repository contents to transition your
 package environment to new version.`,
 		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
 		Subcommands: []*commander.Command{
+			makeCmdConfig(),
 			makeCmdDb(),
 			makeCmdGraph(),
 			makeCmdMirror(),
 			makeCmdRepo(),
 			makeCmdServe(),
 			makeCmdSnapshot(),
-			// Disabled on no docs
+			makeCmdTask(),
-			//makeCmdTask(),
 			makeCmdPublish(),
 			makeCmdVersion(),
 			makeCmdPackage(),
+			makeCmdAPI(),
 		},
 	}
 

cmd/config.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdConfig() *commander.Command {
+	return &commander.Command{
+		UsageLine: "config",
+		Short:     "manage aptly configuration",
+		Subcommands: []*commander.Command{
+			makeCmdConfigShow(),
+		},
+	}
+}

cmd/config_show.go

@@ -0,0 +1,38 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/smira/commander"
+)
+
+func aptlyConfigShow(cmd *commander.Command, args []string) error {
+
+	config := context.Config()
+	prettyJSON, err := json.MarshalIndent(config, "", "    ")
+
+	if err != nil {
+		return fmt.Errorf("unable to dump the config file: %s", err)
+	}
+
+	fmt.Println(string(prettyJSON))
+
+	return nil
+}
+
+func makeCmdConfigShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyConfigShow,
+		UsageLine: "show",
+		Short:     "show current aptly's config",
+		Long: `
+Command show displays the current aptly configuration.
+
+Example:
+
+  $ aptly config show
+
+`,
+	}
+	return cmd
+}

cmd/context.go

@@ -1,313 +1,20 @@
 package cmd
 
 import (
-	"fmt"
+	ctx "github.com/smira/aptly/context"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/console"
-	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/files"
-	"github.com/smira/aptly/http"
-	"github.com/smira/aptly/s3"
-	"github.com/smira/aptly/utils"
-	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"path/filepath"
-	"runtime"
-	"runtime/pprof"
-	"strings"
-	"time"
 )
 
-// AptlyContext is a common context shared by all commands
+var context *ctx.AptlyContext
-type AptlyContext struct {
-	flags, globalFlags *flag.FlagSet
-	configLoaded       bool
-
-	progress          aptly.Progress
-	downloader        aptly.Downloader
-	database          database.Storage
-	packagePool       aptly.PackagePool
-	publishedStorages map[string]aptly.PublishedStorage
-	collectionFactory *deb.CollectionFactory
-	dependencyOptions int
-	architecturesList []string
-	// Debug features
-	fileCPUProfile *os.File
-	fileMemProfile *os.File
-	fileMemStats   *os.File
-}
-
-var context *AptlyContext
-
-// Check interface
-var _ aptly.PublishedStorageProvider = &AptlyContext{}
-
-// FatalError is type for panicking to abort execution with non-zero
-// exit code and print meaningful explanation
-type FatalError struct {
-	ReturnCode int
-	Message    string
-}
-
-// Fatal panics and aborts execution with exit code 1
-func Fatal(err error) {
-	returnCode := 1
-	if err == commander.ErrFlagError || err == commander.ErrCommandError {
-		returnCode = 2
-	}
-	panic(&FatalError{ReturnCode: returnCode, Message: err.Error()})
-}
-
-// Config loads and returns current configuration
-func (context *AptlyContext) Config() *utils.ConfigStructure {
-	if !context.configLoaded {
-		var err error
-
-		configLocation := context.globalFlags.Lookup("config").Value.String()
-		if configLocation != "" {
-			err = utils.LoadConfig(configLocation, &utils.Config)
-
-			if err != nil {
-				Fatal(err)
-			}
-		} else {
-			configLocations := []string{
-				filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
-				"/etc/aptly.conf",
-			}
-
-			for _, configLocation := range configLocations {
-				err = utils.LoadConfig(configLocation, &utils.Config)
-				if err == nil {
-					break
-				}
-				if !os.IsNotExist(err) {
-					Fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
-				}
-			}
-
-			if err != nil {
-				fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
-				utils.SaveConfig(configLocations[0], &utils.Config)
-			}
-		}
-
-		context.configLoaded = true
-
-	}
-	return &utils.Config
-}
-
-// DependencyOptions calculates options related to dependecy handling
-func (context *AptlyContext) DependencyOptions() int {
-	if context.dependencyOptions == -1 {
-		context.dependencyOptions = 0
-		if LookupOption(context.Config().DepFollowSuggests, context.globalFlags, "dep-follow-suggests") {
-			context.dependencyOptions |= deb.DepFollowSuggests
-		}
-		if LookupOption(context.Config().DepFollowRecommends, context.globalFlags, "dep-follow-recommends") {
-			context.dependencyOptions |= deb.DepFollowRecommends
-		}
-		if LookupOption(context.Config().DepFollowAllVariants, context.globalFlags, "dep-follow-all-variants") {
-			context.dependencyOptions |= deb.DepFollowAllVariants
-		}
-		if LookupOption(context.Config().DepFollowSource, context.globalFlags, "dep-follow-source") {
-			context.dependencyOptions |= deb.DepFollowSource
-		}
-	}
-
-	return context.dependencyOptions
-}
-
-// ArchitecturesList returns list of architectures fixed via command line or config
-func (context *AptlyContext) ArchitecturesList() []string {
-	if context.architecturesList == nil {
-		context.architecturesList = context.Config().Architectures
-		optionArchitectures := context.globalFlags.Lookup("architectures").Value.String()
-		if optionArchitectures != "" {
-			context.architecturesList = strings.Split(optionArchitectures, ",")
-		}
-	}
-
-	return context.architecturesList
-}
-
-// Progress creates or returns Progress object
-func (context *AptlyContext) Progress() aptly.Progress {
-	if context.progress == nil {
-		context.progress = console.NewProgress()
-		context.progress.Start()
-	}
-
-	return context.progress
-}
-
-// Downloader returns instance of current downloader
-func (context *AptlyContext) Downloader() aptly.Downloader {
-	if context.downloader == nil {
-		var downloadLimit int64
-		limitFlag := context.flags.Lookup("download-limit")
-		if limitFlag != nil {
-			downloadLimit = limitFlag.Value.Get().(int64)
-		}
-		if downloadLimit == 0 {
-			downloadLimit = context.Config().DownloadLimit
-		}
-		context.downloader = http.NewDownloader(context.Config().DownloadConcurrency,
-			downloadLimit*1024, context.Progress())
-	}
-
-	return context.downloader
-}
-
-// DBPath builds path to database
-func (context *AptlyContext) DBPath() string {
-	return filepath.Join(context.Config().RootDir, "db")
-}
-
-// Database opens and returns current instance of database
-func (context *AptlyContext) Database() (database.Storage, error) {
-	if context.database == nil {
-		var err error
-
-		context.database, err = database.OpenDB(context.DBPath())
-		if err != nil {
-			return nil, fmt.Errorf("can't open database: %s", err)
-		}
-	}
-
-	return context.database, nil
-}
-
-// CloseDatabase closes the db temporarily
-func (context *AptlyContext) CloseDatabase() error {
-	if context.database == nil {
-		return nil
-	}
-
-	return context.database.Close()
-}
-
-// ReOpenDatabase reopens the db after close
-func (context *AptlyContext) ReOpenDatabase() error {
-	if context.database == nil {
-		return nil
-	}
-
-	const MaxTries = 10
-	const Delay = 10 * time.Second
-
-	for try := 0; try < MaxTries; try++ {
-		err := context.database.ReOpen()
-		if err == nil || strings.Index(err.Error(), "resource temporarily unavailable") == -1 {
-			return err
-		}
-		context.Progress().Printf("Unable to reopen database, sleeping %s\n", Delay)
-		<-time.After(Delay)
-	}
-
-	return fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
-}
-
-// CollectionFactory builds factory producing all kinds of collections
-func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
-	if context.collectionFactory == nil {
-		db, err := context.Database()
-		if err != nil {
-			Fatal(err)
-		}
-		context.collectionFactory = deb.NewCollectionFactory(db)
-	}
-
-	return context.collectionFactory
-}
-
-// PackagePool returns instance of PackagePool
-func (context *AptlyContext) PackagePool() aptly.PackagePool {
-	if context.packagePool == nil {
-		context.packagePool = files.NewPackagePool(context.Config().RootDir)
-	}
-
-	return context.packagePool
-}
-
-// GetPublishedStorage returns instance of PublishedStorage
-func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
-	publishedStorage, ok := context.publishedStorages[name]
-	if !ok {
-		if name == "" {
-			publishedStorage = files.NewPublishedStorage(context.Config().RootDir)
-		} else if strings.HasPrefix(name, "s3:") {
-			params, ok := context.Config().S3PublishRoots[name[3:]]
-			if !ok {
-				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
-			}
-
-			var err error
-			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
-				params.Region, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
-				params.EncryptionMethod, params.PlusWorkaround)
-			if err != nil {
-				Fatal(err)
-			}
-		} else {
-			Fatal(fmt.Errorf("unknown published storage format: %v", name))
-		}
-		context.publishedStorages[name] = publishedStorage
-	}
-
-	return publishedStorage
-}
-
-// UpdateFlags sets internal copy of flags in the context
-func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) {
-	context.flags = flags
-}
 
 // ShutdownContext shuts context down
 func ShutdownContext() {
-	if aptly.EnableDebug {
+	context.Shutdown()
-		if context.fileMemProfile != nil {
-			pprof.WriteHeapProfile(context.fileMemProfile)
-			context.fileMemProfile.Close()
-			context.fileMemProfile = nil
-		}
-		if context.fileCPUProfile != nil {
-			pprof.StopCPUProfile()
-			context.fileCPUProfile.Close()
-			context.fileCPUProfile = nil
-		}
-		if context.fileMemProfile != nil {
-			context.fileMemProfile.Close()
-			context.fileMemProfile = nil
-		}
-	}
-	if context.database != nil {
-		context.database.Close()
-		context.database = nil
-	}
-	if context.downloader != nil {
-		context.downloader.Abort()
-		context.downloader = nil
-	}
-	if context.progress != nil {
-		context.progress.Shutdown()
-		context.progress = nil
-	}
 }
 
 // CleanupContext does partial shutdown of context
 func CleanupContext() {
-	if context.downloader != nil {
+	context.Cleanup()
-		context.downloader.Shutdown()
-		context.downloader = nil
-	}
-	if context.progress != nil {
-		context.progress.Shutdown()
-		context.progress = nil
-	}
 }
 
 // InitContext initializes context with default settings
@@ -318,60 +25,7 @@ func InitContext(flags *flag.FlagSet) error {
 		panic("context already initialized")
 	}
 
-	context = &AptlyContext{
+	context, err = ctx.NewContext(flags)
-		flags:             flags,
-		globalFlags:       flags,
-		dependencyOptions: -1,
-		publishedStorages: map[string]aptly.PublishedStorage{},
-	}
 
-	if aptly.EnableDebug {
+	return err
-		cpuprofile := flags.Lookup("cpuprofile").Value.String()
-		if cpuprofile != "" {
-			context.fileCPUProfile, err = os.Create(cpuprofile)
-			if err != nil {
-				return err
-			}
-			pprof.StartCPUProfile(context.fileCPUProfile)
-		}
-
-		memprofile := flags.Lookup("memprofile").Value.String()
-		if memprofile != "" {
-			context.fileMemProfile, err = os.Create(memprofile)
-			if err != nil {
-				return err
-			}
-		}
-
-		memstats := flags.Lookup("memstats").Value.String()
-		if memstats != "" {
-			interval := flags.Lookup("meminterval").Value.Get().(time.Duration)
-
-			context.fileMemStats, err = os.Create(memstats)
-			if err != nil {
-				return err
-			}
-
-			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
-
-			go func() {
-				var stats runtime.MemStats
-
-				start := time.Now().UnixNano()
-
-				for {
-					runtime.ReadMemStats(&stats)
-					if context.fileMemStats != nil {
-						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
-							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
-						time.Sleep(interval)
-					} else {
-						break
-					}
-				}
-			}()
-		}
-	}
-
-	return nil
 }

cmd/db_cleanup.go

@@ -20,14 +20,14 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	// collect information about references packages...
 	existingPackageRefs := deb.NewPackageRefList()
 
-	context.Progress().Printf("Loading mirrors, local repos and snapshots...\n")
+	context.Progress().Printf("Loading mirrors, local repos, snapshots and published repos...\n")
 	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
 		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
 		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
 		}
 		return nil
 	})
@@ -41,7 +41,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			return err
 		}
 		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
 		}
 		return nil
 	})
@@ -54,7 +54,25 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		if err != nil {
 			return err
 		}
-		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false)
+		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+		if published.SourceKind != "local" {
+			return nil
+		}
+		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+		if err != nil {
+			return err
+		}
+
+		for _, component := range published.Components() {
+			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+		}
 		return nil
 	})
 	if err != nil {

cmd/graph.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"bytes"
-	"code.google.com/p/gographviz"
 	"fmt"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
@@ -10,7 +9,6 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"strings"
 )
 
 func aptlyGraph(cmd *commander.Command, args []string) error {
@@ -21,121 +19,11 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	graph := gographviz.NewEscape()
-	graph.SetDir(true)
-	graph.SetName("aptly")
-
-	existingNodes := map[string]bool{}
-
-	fmt.Printf("Loading mirrors...\n")
-
-	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
-		}
-
-		graph.AddNode("aptly", repo.UUID, map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "darkgoldenrod1",
-			"label": fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
-				repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "),
-				strings.Join(repo.Architectures, ", "), repo.NumPackages()),
-		})
-		existingNodes[repo.UUID] = true
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Loading local repos...\n")
-
-	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
-		}
-
-		graph.AddNode("aptly", repo.UUID, map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "mediumseagreen",
-			"label": fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
-				repo.Name, repo.Comment, repo.NumPackages()),
-		})
-		existingNodes[repo.UUID] = true
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Loading snapshots...\n")
-
-	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		existingNodes[snapshot.UUID] = true
-		return nil
-	})
-
-	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
-			return err
-		}
-
-		description := snapshot.Description
-		if snapshot.SourceKind == "repo" {
-			description = "Snapshot from repo"
-		}
-
-		graph.AddNode("aptly", snapshot.UUID, map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "cadetblue1",
-			"label":     fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages()),
-		})
-
-		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
-			for _, uuid := range snapshot.SourceIDs {
-				_, exists := existingNodes[uuid]
-				if exists {
-					graph.AddEdge(uuid, snapshot.UUID, true, nil)
-				}
-			}
-		}
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Loading published repos...\n")
-
-	context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		graph.AddNode("aptly", repo.UUID, map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "darkolivegreen1",
-			"label": fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution,
-				strings.Join(repo.Components(), " "), strings.Join(repo.Architectures, ", ")),
-		})
-
-		for _, uuid := range repo.Sources {
-			_, exists := existingNodes[uuid]
-			if exists {
-				graph.AddEdge(uuid, repo.UUID, true, nil)
-			}
-		}
-
-		return nil
-	})
-
 	fmt.Printf("Generating graph...\n")
+	graph, err := deb.BuildGraph(context.CollectionFactory())
+	if err != nil {
+		return err
+	}
 
 	buf := bytes.NewBufferString(graph.String())
 

cmd/mirror_create.go

@@ -16,8 +16,8 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.flags, "with-sources")
+	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
-	downloadUdebs := context.flags.Lookup("with-udebs").Value.Get().(bool)
+	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -40,8 +40,9 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
-	repo.Filter = context.flags.Lookup("filter").Value.String()
+	repo.Filter = context.Flags().Lookup("filter").Value.String()
-	repo.FilterWithDeps = context.flags.Lookup("filter-with-deps").Value.Get().(bool)
+	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
+	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
 
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
@@ -50,7 +51,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	verifier, err := getVerifier(context.flags)
+	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
@@ -95,6 +96,7 @@ Example:
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/mirror_drop.go

@@ -25,7 +25,7 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
 		snapshots := context.CollectionFactory().SnapshotCollection().ByRemoteRepoSource(repo)
 

cmd/mirror_edit.go

@@ -24,7 +24,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	context.flags.Visit(func(flag *flag.Flag) {
+	context.Flags().Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
 			repo.Filter = flag.Value.String()
@@ -48,7 +48,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	if context.globalFlags.Lookup("architectures").Value.String() != "" {
+	if context.GlobalFlags().Lookup("architectures").Value.String() != "" {
 		repo.Architectures = context.ArchitecturesList()
 
 		err = repo.Fetch(context.Downloader(), nil)

cmd/mirror_list.go

@@ -28,6 +28,8 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
+	context.CloseDatabase()
+
 	sort.Strings(repos)
 
 	if raw {

cmd/mirror_show.go

@@ -66,7 +66,7 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		fmt.Printf("%s: %s\n", k, repo.Meta[k])
 	}
 
-	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		if repo.LastDownloadDate.IsZero() {
 			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")

cmd/mirror_update.go

@@ -31,7 +31,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
 		err = repo.CheckLock()
 		if err != nil {
@@ -39,9 +39,9 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	ignoreMismatch := context.flags.Lookup("ignore-checksums").Value.Get().(bool)
+	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
 
-	verifier, err := getVerifier(context.flags)
+	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}

cmd/package_search.go

@@ -21,6 +21,10 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 	}
 
 	result := q.Query(context.CollectionFactory().PackageCollection())
+	if result.Len() == 0 {
+		return fmt.Errorf("no results")
+	}
+
 	result.ForEach(func(p *deb.Package) error {
 		context.Progress().Printf("%s\n", p)
 		return nil

cmd/package_show.go

@@ -72,15 +72,15 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	withFiles := context.flags.Lookup("with-files").Value.Get().(bool)
+	withFiles := context.Flags().Lookup("with-files").Value.Get().(bool)
-	withReferences := context.flags.Lookup("with-references").Value.Get().(bool)
+	withReferences := context.Flags().Lookup("with-references").Value.Get().(bool)
 
 	w := bufio.NewWriter(os.Stdout)
 
 	result := q.Query(context.CollectionFactory().PackageCollection())
 
 	err = result.ForEach(func(p *deb.Package) error {
-		p.Stanza().WriteTo(w)
+		p.Stanza().WriteTo(w, p.IsSource, false)
 		w.Flush()
 		fmt.Printf("\n")
 
@@ -116,7 +116,7 @@ func makeCmdPackageShow() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPackageShow,
 		UsageLine: "show <package-query>",
-		Short:     "show details about packages matcing query",
+		Short:     "show details about packages matching query",
 		Long: `
 Command shows displays detailed meta-information about packages
 matching query. Information from Debian control file is displayed.

cmd/publish.go

@@ -4,7 +4,6 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
@@ -16,6 +15,7 @@ func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
 	signer.SetKey(flags.Lookup("gpg-key").Value.String())
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
 	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
+	signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
 
 	err := signer.Init()
 	if err != nil {
@@ -26,20 +26,6 @@ func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
 
 }
 
-func parsePrefix(param string) (storage, prefix string) {
-	i := strings.LastIndex(param, ":")
-	if i != -1 {
-		storage = param[:i]
-		prefix = param[i+1:]
-		if prefix == "" {
-			prefix = "."
-		}
-	} else {
-		prefix = param
-	}
-	return
-}
-
 func makeCmdPublish() *commander.Command {
 	return &commander.Command{
 		UsageLine: "publish",

cmd/publish_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -19,7 +20,7 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 		param = args[1]
 	}
 
-	storage, prefix := parsePrefix(param)
+	storage, prefix := deb.ParsePrefix(param)
 
 	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
 		context.CollectionFactory(), context.Progress())

cmd/publish_list.go

@@ -36,6 +36,8 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load list of repos: %s", err)
 	}
 
+	context.CloseDatabase()
+
 	sort.Strings(published)
 
 	if raw {

cmd/publish_repo.go

@@ -39,6 +39,7 @@ Example:
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")

cmd/publish_snapshot.go

@@ -13,7 +13,7 @@ import (
 func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	var err error
 
-	components := strings.Split(context.flags.Lookup("component").Value.String(), ",")
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
 
 	if len(args) < len(components) || len(args) > len(components)+1 {
 		cmd.Usage()
@@ -27,7 +27,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	} else {
 		param = ""
 	}
-	storage, prefix := parsePrefix(param)
+	storage, prefix := deb.ParsePrefix(param)
 
 	var (
 		sources = []interface{}{}
@@ -110,7 +110,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		panic("unknown command")
 	}
 
-	distribution := context.flags.Lookup("distribution").Value.String()
+	distribution := context.Flags().Lookup("distribution").Value.String()
 
 	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
 	if err != nil {
@@ -125,12 +125,12 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 	}
 
-	signer, err := getSigner(context.flags)
+	signer, err := getSigner(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
 	if forceOverwrite {
 		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
 			"the same package pool.\n")
@@ -201,6 +201,7 @@ Example:
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")

cmd/publish_switch.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"strings"
@@ -11,7 +12,7 @@ import (
 func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	var err error
 
-	components := strings.Split(context.flags.Lookup("component").Value.String(), ",")
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
 
 	if len(args) < len(components)+1 || len(args) > len(components)+2 {
 		cmd.Usage()
@@ -33,7 +34,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		names = args[1:]
 	}
 
-	storage, prefix := parsePrefix(param)
+	storage, prefix := deb.ParsePrefix(param)
 
 	var published *deb.PublishedRepo
 
@@ -61,6 +62,10 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	}
 
 	for i, component := range components {
+		if !utils.StrSliceHasItem(publishedComponents, component) {
+			return fmt.Errorf("unable to switch: component %s is not in published repository", component)
+		}
+
 		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(names[i])
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
@@ -74,12 +79,12 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		published.UpdateSnapshot(component, snapshot)
 	}
 
-	signer, err := getSigner(context.flags)
+	signer, err := getSigner(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
 	if forceOverwrite {
 		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
 			"the same package pool.\n")
@@ -112,7 +117,7 @@ func makeCmdPublishSwitch() *commander.Command {
 		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
 		Short:     "update published repository by switching to new snapshot",
 		Long: `
-Command switches in-place published repository with new snapshot contents. All
+Command switches in-place published snapshots with new snapshot contents. All
 publishing parameters are preserved (architecture list, distribution,
 component).
 
@@ -120,11 +125,14 @@ For multiple component repositories, flag -component should be given with
 list of components to update. Corresponding snapshots should be given in the
 same order, e.g.:
 
-	aptly publish update -component=main,contrib wheezy wh-main wh-contrib
+	aptly publish switch -component=main,contrib wheezy wh-main wh-contrib
 
 Example:
 
-    $ aptly publish update wheezy ppa wheezy-7.5
+    $ aptly publish switch wheezy ppa wheezy-7.5
+
+This command would switch published repository (with one component) named ppa/wheezy
+(prefix ppa, dsitribution wheezy to new snapshot wheezy-7.5).
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
 	}
@@ -133,6 +141,7 @@ Example:
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")

cmd/publish_update.go

@@ -20,7 +20,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 	if len(args) == 2 {
 		param = args[1]
 	}
-	storage, prefix := parsePrefix(param)
+	storage, prefix := deb.ParsePrefix(param)
 
 	var published *deb.PublishedRepo
 
@@ -43,12 +43,12 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		published.UpdateLocalRepo(component)
 	}
 
-	signer, err := getSigner(context.flags)
+	signer, err := getSigner(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
 	if forceOverwrite {
 		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
 			"the same package pool.\n")
@@ -100,6 +100,7 @@ Example:
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 

cmd/repo_add.go

@@ -2,14 +2,12 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"os"
-	"path/filepath"
-	"sort"
-	"strings"
 )
 
 func aptlyRepoAdd(cmd *commander.Command, args []string) error {
@@ -40,151 +38,22 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	forceReplace := context.flags.Lookup("force-replace").Value.Get().(bool)
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 
-	packageFiles := []string{}
+	var packageFiles, failedFiles []string
-	failedFiles := []string{}
 
-	for _, location := range args[1:] {
+	packageFiles, failedFiles, err = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
-		info, err2 := os.Stat(location)
+	if err != nil {
-		if err2 != nil {
+		return fmt.Errorf("unable to collect package files: %s", err)
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to process %s: %s@|", location, err2)
-			failedFiles = append(failedFiles, location)
-			continue
-		}
-		if info.IsDir() {
-			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
-				if err3 != nil {
-					return err3
-				}
-				if info.IsDir() {
-					return nil
-				}
-
-				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
-					strings.HasSuffix(info.Name(), ".dsc") {
-					packageFiles = append(packageFiles, path)
-				}
-
-				return nil
-			})
-		} else {
-			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
-				strings.HasSuffix(info.Name(), ".dsc") {
-				packageFiles = append(packageFiles, location)
-			} else {
-				context.Progress().ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
-				failedFiles = append(failedFiles, location)
-				continue
-			}
-		}
 	}
 
-	processedFiles := []string{}
+	var processedFiles, failedFiles2 []string
-	sort.Strings(packageFiles)
 
-	if forceReplace {
+	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		list.PrepareIndex()
+		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()})
-	}
+	failedFiles = append(failedFiles, failedFiles2...)
-
+	if err != nil {
-	for _, file := range packageFiles {
+		return fmt.Errorf("unable to import package files: %s", err)
-		var (
-			stanza deb.Stanza
-			p      *deb.Package
-		)
-
-		candidateProcessedFiles := []string{}
-		isSourcePackage := strings.HasSuffix(file, ".dsc")
-		isUdebPackage := strings.HasSuffix(file, ".udeb")
-
-		if isSourcePackage {
-			stanza, err = deb.GetControlFileFromDsc(file, verifier)
-
-			if err == nil {
-				stanza["Package"] = stanza["Source"]
-				delete(stanza, "Source")
-
-				p, err = deb.NewSourcePackageFromControlFile(stanza)
-			}
-		} else {
-			stanza, err = deb.GetControlFileFromDeb(file)
-			if isUdebPackage {
-				p = deb.NewUdebPackageFromControlFile(stanza)
-			} else {
-				p = deb.NewPackageFromControlFile(stanza)
-			}
-		}
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
-			failedFiles = append(failedFiles, file)
-			continue
-		}
-
-		var checksums utils.ChecksumInfo
-		checksums, err = utils.ChecksumsForFile(file)
-		if err != nil {
-			return err
-		}
-
-		if isSourcePackage {
-			p.UpdateFiles(append(p.Files(), deb.PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
-		} else {
-			p.UpdateFiles([]deb.PackageFile{deb.PackageFile{Filename: filepath.Base(file), Checksums: checksums}})
-		}
-
-		err = context.PackagePool().Import(file, checksums.MD5)
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", file, err)
-			failedFiles = append(failedFiles, file)
-			continue
-		}
-
-		candidateProcessedFiles = append(candidateProcessedFiles, file)
-
-		// go over all files, except for the last one (.dsc/.deb itself)
-		for _, f := range p.Files() {
-			if filepath.Base(f.Filename) == filepath.Base(file) {
-				continue
-			}
-			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
-			err = context.PackagePool().Import(sourceFile, f.Checksums.MD5)
-			if err != nil {
-				context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", sourceFile, err)
-				failedFiles = append(failedFiles, file)
-				break
-			}
-
-			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
-		}
-		if err != nil {
-			// some files haven't been imported
-			continue
-		}
-
-		err = context.CollectionFactory().PackageCollection().Update(p)
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to save package %s: %s@|", p, err)
-			failedFiles = append(failedFiles, file)
-			continue
-		}
-
-		if forceReplace {
-			conflictingPackages := list.Search(deb.Dependency{Pkg: p.Name, Version: p.Version, Architecture: p.Architecture}, true)
-			for _, cp := range conflictingPackages {
-				context.Progress().ColoredPrintf("@r[-]@| %s removed due to conflict with package being added", cp)
-				list.Remove(cp)
-			}
-		}
-
-		err = list.Add(p)
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
-			failedFiles = append(failedFiles, file)
-			continue
-		}
-
-		context.Progress().ColoredPrintf("@g[+]@| %s added@|", p)
-		processedFiles = append(processedFiles, candidateProcessedFiles...)
 	}
 
 	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
@@ -194,7 +63,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save: %s", err)
 	}
 
-	if context.flags.Lookup("remove-files").Value.Get().(bool) {
+	if context.Flags().Lookup("remove-files").Value.Get().(bool) {
 		processedFiles = utils.StrSliceDeduplicate(processedFiles)
 
 		for _, file := range processedFiles {

cmd/repo_create.go

@@ -14,9 +14,9 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	repo := deb.NewLocalRepo(args[0], context.flags.Lookup("comment").Value.String())
+	repo := deb.NewLocalRepo(args[0], context.Flags().Lookup("comment").Value.String())
-	repo.DefaultDistribution = context.flags.Lookup("distribution").Value.String()
+	repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
-	repo.DefaultComponent = context.flags.Lookup("component").Value.String()
+	repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
 
 	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
 	if err != nil {

cmd/repo_drop.go

@@ -34,7 +34,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: local repo is published")
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
 		snapshots := context.CollectionFactory().SnapshotCollection().ByLocalRepoSource(repo)
 

cmd/repo_edit.go

@@ -23,16 +23,16 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	if context.flags.Lookup("comment").Value.String() != "" {
+	if context.Flags().Lookup("comment").Value.String() != "" {
-		repo.Comment = context.flags.Lookup("comment").Value.String()
+		repo.Comment = context.Flags().Lookup("comment").Value.String()
 	}
 
-	if context.flags.Lookup("distribution").Value.String() != "" {
+	if context.Flags().Lookup("distribution").Value.String() != "" {
-		repo.DefaultDistribution = context.flags.Lookup("distribution").Value.String()
+		repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
 	}
 
-	if context.flags.Lookup("component").Value.String() != "" {
+	if context.Flags().Lookup("component").Value.String() != "" {
-		repo.DefaultComponent = context.flags.Lookup("component").Value.String()
+		repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
 	}
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)

cmd/repo_list.go

@@ -33,6 +33,8 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
+	context.CloseDatabase()
+
 	sort.Strings(repos)
 
 	if raw {

cmd/repo_move.go

@@ -87,7 +87,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	var architecturesList []string
 
-	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
 
 	if withDeps {
 		dstList.PrepareIndex()
@@ -145,7 +145,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
 		dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))

cmd/repo_remove.go

@@ -54,7 +54,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
-	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
 		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))

cmd/repo_show.go

@@ -31,7 +31,7 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Default Component: %s\n", repo.DefaultComponent)
 	fmt.Printf("Number of packages: %d\n", repo.NumPackages())
 
-	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		ListPackagesRefList(repo.RefList())
 	}

cmd/run.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	ctx "github.com/smira/aptly/context"
 	"github.com/smira/commander"
 )
 
@@ -9,7 +10,7 @@ import (
 func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode int) {
 	defer func() {
 		if r := recover(); r != nil {
-			fatal, ok := r.(*FatalError)
+			fatal, ok := r.(*ctx.FatalError)
 			if !ok {
 				panic(r)
 			}
@@ -22,13 +23,13 @@ func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode
 
 	flags, args, err := cmd.ParseFlags(cmdArgs)
 	if err != nil {
-		Fatal(err)
+		ctx.Fatal(err)
 	}
 
 	if initContext {
 		err = InitContext(flags)
 		if err != nil {
-			Fatal(err)
+			ctx.Fatal(err)
 		}
 		defer ShutdownContext()
 	}
@@ -37,7 +38,7 @@ func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode
 
 	err = cmd.Dispatch(args)
 	if err != nil {
-		Fatal(err)
+		ctx.Fatal(err)
 	}
 
 	return

cmd/serve.go

@@ -27,7 +27,7 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		return nil
 	}
 
-	listen := context.flags.Lookup("listen").Value.String()
+	listen := context.Flags().Lookup("listen").Value.String()
 
 	listenHost, listenPort, err := net.SplitHostPort(listen)
 

cmd/snapshot_diff.go

@@ -13,7 +13,7 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	onlyMatching := context.flags.Lookup("only-matching").Value.Get().(bool)
+	onlyMatching := context.Flags().Lookup("only-matching").Value.Get().(bool)
 
 	// Load <name-a> snapshot
 	snapshotA, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])

cmd/snapshot_drop.go

@@ -35,7 +35,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: snapshot is published")
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
 		snapshots := context.CollectionFactory().SnapshotCollection().BySnapshotSource(snapshot)
 		if len(snapshots) > 0 {

cmd/snapshot_filter.go

@@ -17,7 +17,7 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
 
 	// Load <source> snapshot
 	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])

cmd/snapshot_list.go

@@ -4,49 +4,8 @@ import (
 	"fmt"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
-	"sort"
 )
 
-// Snapshot sorting methods
-const (
-	SortName = iota
-	SortTime
-)
-
-type snapshotListToSort struct {
-	list       []*deb.Snapshot
-	sortMethod int
-}
-
-func parseSortMethod(sortMethod string) (int, error) {
-	switch sortMethod {
-	case "time", "Time":
-		return SortTime, nil
-	case "name", "Name":
-		return SortName, nil
-	}
-
-	return -1, fmt.Errorf("sorting method \"%s\" unknown", sortMethod)
-}
-
-func (s snapshotListToSort) Swap(i, j int) {
-	s.list[i], s.list[j] = s.list[j], s.list[i]
-}
-
-func (s snapshotListToSort) Less(i, j int) bool {
-	switch s.sortMethod {
-	case SortName:
-		return s.list[i].Name < s.list[j].Name
-	case SortTime:
-		return s.list[i].CreatedAt.Before(s.list[j].CreatedAt)
-	}
-	panic("unknown sort method")
-}
-
-func (s snapshotListToSort) Len() int {
-	return len(s.list)
-}
-
 func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
@@ -57,33 +16,24 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
 
-	snapshotsToSort := &snapshotListToSort{}
+	collection := context.CollectionFactory().SnapshotCollection()
-	snapshotsToSort.list = make([]*deb.Snapshot, context.CollectionFactory().SnapshotCollection().Len())
-	snapshotsToSort.sortMethod, err = parseSortMethod(sortMethodString)
-	if err != nil {
-		return err
-	}
-
-	i := 0
-	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		snapshotsToSort.list[i] = snapshot
-		i++
-
-		return nil
-	})
-
-	sort.Sort(snapshotsToSort)
 
 	if raw {
-		for _, snapshot := range snapshotsToSort.list {
+		collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
 			fmt.Printf("%s\n", snapshot.Name)
-		}
+			return nil
+		})
 	} else {
-		if len(snapshotsToSort.list) > 0 {
+		if collection.Len() > 0 {
 			fmt.Printf("List of snapshots:\n")
 
-			for _, snapshot := range snapshotsToSort.list {
+			err = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
 				fmt.Printf(" * %s\n", snapshot.String())
+				return nil
+			})
+
+			if err != nil {
+				return err
 			}
 
 			fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
@@ -91,8 +41,8 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
 			fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n")
 		}
 	}
-	return err
 
+	return err
 }
 
 func makeCmdSnapshotList() *commander.Command {

cmd/snapshot_merge.go

@@ -28,22 +28,22 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	latest := context.flags.Lookup("latest").Value.Get().(bool)
+	latest := context.Flags().Lookup("latest").Value.Get().(bool)
-    noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
+	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
 
-    if noRemove && latest {
+	if noRemove && latest {
-        return fmt.Errorf("-no-remove and -latest can't be specified together")
+		return fmt.Errorf("-no-remove and -latest can't be specified together")
-    }
+	}
 
 	overrideMatching := !latest && !noRemove
 
 	result := sources[0].RefList()
 	for i := 1; i < len(sources); i++ {
-		result = result.Merge(sources[i].RefList(), overrideMatching)
+		result = result.Merge(sources[i].RefList(), overrideMatching, false)
 	}
 
 	if latest {
-		deb.FilterLatestRefs(result)
+		result.FilterLatestRefs()
 	}
 
 	sourceDescription := make([]string, len(sources))
@@ -84,7 +84,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("latest", false, "use only the latest version of each package")
-    cmd.Flag.Bool("no-remove", false, "don't remove duplicate arch/name packages")
+	cmd.Flag.Bool("no-remove", false, "don't remove duplicate arch/name packages")
 
 	return cmd
 }

cmd/snapshot_pull.go

@@ -17,9 +17,9 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	noDeps := context.flags.Lookup("no-deps").Value.Get().(bool)
+	noDeps := context.Flags().Lookup("no-deps").Value.Get().(bool)
-	noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
+	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
-	allMatches := context.flags.Lookup("all-matches").Value.Get().(bool)
+	allMatches := context.Flags().Lookup("all-matches").Value.Get().(bool)
 
 	// Load <name> snapshot
 	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
@@ -91,7 +91,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to parse query: %s", err)
 		}
 		// Add architecture filter
-		queries[i] = &deb.AndQuery{queries[i], archQuery}
+		queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
 	}
 
 	// Filter with dependencies as requested
@@ -129,7 +129,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	})
 	alreadySeen = nil
 
-	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nNot creating snapshot, as dry run was requested.\n")
 	} else {
 		// Create <destination> snapshot

cmd/snapshot_search.go

@@ -73,7 +73,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		return fmt.Errorf("unable to search: %s", err)
 	}
 
-	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
 	architecturesList := []string{}
 
 	if withDeps {

cmd/snapshot_show.go

@@ -30,7 +30,7 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Description: %s\n", snapshot.Description)
 	fmt.Printf("Number of packages: %d\n", snapshot.NumPackages())
 
-	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		ListPackagesRefList(snapshot.RefList())
 	}

cmd/task_run.go

@@ -18,13 +18,15 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 		var text string
 		cmdArgs := []string{}
 
-		if finfo, err := os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
+		var finfo os.FileInfo
+		if finfo, err = os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
 			return fmt.Errorf("no such file, %s\n", filename)
 		}
 
-		fmt.Println("Reading file...\n")
+		fmt.Print("Reading file...\n\n")
 
-		file, err := os.Open(filename)
+		var file *os.File
+		file, err = os.Open(filename)
 
 		if err != nil {
 			return err
@@ -80,6 +82,10 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 
 	for i, command := range cmdList {
 		if !commandErrored {
+			err := context.ReOpenDatabase()
+			if err != nil {
+				return fmt.Errorf("failed to reopen DB: %s", err)
+			}
 			context.Progress().ColoredPrintf("@g%d) [Running]: %s@!", (i + 1), strings.Join(command, " "))
 			context.Progress().ColoredPrintf("\n@yBegin command output: ----------------------------@!")
 			context.Progress().Flush()
@@ -129,16 +135,16 @@ func makeCmdTaskRun() *commander.Command {
 		UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
 		Short:     "run aptly tasks",
 		Long: `
-Command helps origanise multiple aptly commands in one single aptly task, running as single thread.
+Command helps organise multiple aptly commands in one single aptly task, running as single thread.
 
 Example:
 
-  $ aptly task run
+	  $ aptly task run
-  > repo create local
+	  > repo create local
-  > repo add local pkg1
+	  > repo add local pkg1
-  > publish repo local
+	  > publish repo local
-  > serve
+	  > serve
-  >
+	  >
 
 `,
 	}

console/progress.go

@@ -14,6 +14,8 @@ const (
 	codeHideProgress
 	codeStop
 	codeFlush
+	codeBarEnabled
+	codeBarDisabled
 )
 
 type printTask struct {
@@ -81,6 +83,8 @@ func (p *Progress) InitBar(count int64, isBytes bool) {
 			p.bar.SetUnits(pb.U_BYTES)
 			p.bar.ShowSpeed = true
 		}
+
+		p.queue <- printTask{code: codeBarEnabled}
 		p.bar.Start()
 	}
 }
@@ -91,6 +95,7 @@ func (p *Progress) ShutdownBar() {
 		return
 	}
 	p.bar.Finish()
+	p.queue <- printTask{code: codeBarDisabled}
 	p.bar = nil
 	p.queue <- printTask{code: codeHideProgress}
 }
@@ -151,9 +156,15 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
 }
 
 func (p *Progress) worker() {
+	hasBar := false
+
 	for {
 		task := <-p.queue
 		switch task.code {
+		case codeBarEnabled:
+			hasBar = true
+		case codeBarDisabled:
+			hasBar = false
 		case codePrint:
 			if p.barShown {
 				fmt.Print("\r\033[2K")
@@ -161,7 +172,7 @@ func (p *Progress) worker() {
 			}
 			fmt.Print(task.message)
 		case codeProgress:
-			if p.bar != nil {
+			if hasBar {
 				fmt.Print("\r" + task.message)
 				p.barShown = true
 			}

console/terminal.go

@@ -1,9 +1,7 @@
-// +build !freebsd
-
 package console
 
 import (
-	"code.google.com/p/go.crypto/ssh/terminal"
+	"golang.org/x/crypto/ssh/terminal"
 	"syscall"
 )
 

console/terminal_bsd.go

@@ -1,10 +0,0 @@
-// +build freebsd
-
-package console
-
-// RunningOnTerminal checks whether stdout is terminal
-//
-// Stub for FreeBSD, until in go1.3 terminal.IsTerminal would start working for FreeBSD
-func RunningOnTerminal() bool {
-	return false
-}

context/context.go

@@ -0,0 +1,490 @@
+// Package context provides single entry to all resources
+package context
+
+import (
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/console"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/files"
+	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/s3"
+	"github.com/smira/aptly/swift"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"strings"
+	"sync"
+	"time"
+)
+
+// AptlyContext is a common context shared by all commands
+type AptlyContext struct {
+	sync.Mutex
+
+	flags, globalFlags *flag.FlagSet
+	configLoaded       bool
+
+	progress          aptly.Progress
+	downloader        aptly.Downloader
+	database          database.Storage
+	packagePool       aptly.PackagePool
+	publishedStorages map[string]aptly.PublishedStorage
+	collectionFactory *deb.CollectionFactory
+	dependencyOptions int
+	architecturesList []string
+	// Debug features
+	fileCPUProfile *os.File
+	fileMemProfile *os.File
+	fileMemStats   *os.File
+}
+
+// Check interface
+var _ aptly.PublishedStorageProvider = &AptlyContext{}
+
+// FatalError is type for panicking to abort execution with non-zero
+// exit code and print meaningful explanation
+type FatalError struct {
+	ReturnCode int
+	Message    string
+}
+
+// Fatal panics and aborts execution with exit code 1
+func Fatal(err error) {
+	returnCode := 1
+	if err == commander.ErrFlagError || err == commander.ErrCommandError {
+		returnCode = 2
+	}
+	panic(&FatalError{ReturnCode: returnCode, Message: err.Error()})
+}
+
+// Config loads and returns current configuration
+func (context *AptlyContext) Config() *utils.ConfigStructure {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.config()
+}
+
+func (context *AptlyContext) config() *utils.ConfigStructure {
+	if !context.configLoaded {
+		var err error
+
+		configLocation := context.globalFlags.Lookup("config").Value.String()
+		if configLocation != "" {
+			err = utils.LoadConfig(configLocation, &utils.Config)
+
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			configLocations := []string{
+				filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
+				"/etc/aptly.conf",
+			}
+
+			for _, configLocation := range configLocations {
+				err = utils.LoadConfig(configLocation, &utils.Config)
+				if err == nil {
+					break
+				}
+				if !os.IsNotExist(err) {
+					Fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
+				}
+			}
+
+			if err != nil {
+				fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
+				utils.SaveConfig(configLocations[0], &utils.Config)
+			}
+		}
+
+		context.configLoaded = true
+
+	}
+	return &utils.Config
+}
+
+// LookupOption checks boolean flag with default (usually config) and command-line
+// setting
+func (context *AptlyContext) LookupOption(defaultValue bool, name string) (result bool) {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.lookupOption(defaultValue, name)
+}
+
+func (context *AptlyContext) lookupOption(defaultValue bool, name string) (result bool) {
+	result = defaultValue
+
+	if context.globalFlags.IsSet(name) {
+		result = context.globalFlags.Lookup(name).Value.Get().(bool)
+	}
+
+	return
+}
+
+// DependencyOptions calculates options related to dependecy handling
+func (context *AptlyContext) DependencyOptions() int {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.dependencyOptions == -1 {
+		context.dependencyOptions = 0
+		if context.lookupOption(context.config().DepFollowSuggests, "dep-follow-suggests") {
+			context.dependencyOptions |= deb.DepFollowSuggests
+		}
+		if context.lookupOption(context.config().DepFollowRecommends, "dep-follow-recommends") {
+			context.dependencyOptions |= deb.DepFollowRecommends
+		}
+		if context.lookupOption(context.config().DepFollowAllVariants, "dep-follow-all-variants") {
+			context.dependencyOptions |= deb.DepFollowAllVariants
+		}
+		if context.lookupOption(context.config().DepFollowSource, "dep-follow-source") {
+			context.dependencyOptions |= deb.DepFollowSource
+		}
+	}
+
+	return context.dependencyOptions
+}
+
+// ArchitecturesList returns list of architectures fixed via command line or config
+func (context *AptlyContext) ArchitecturesList() []string {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.architecturesList == nil {
+		context.architecturesList = context.config().Architectures
+		optionArchitectures := context.globalFlags.Lookup("architectures").Value.String()
+		if optionArchitectures != "" {
+			context.architecturesList = strings.Split(optionArchitectures, ",")
+		}
+	}
+
+	return context.architecturesList
+}
+
+// Progress creates or returns Progress object
+func (context *AptlyContext) Progress() aptly.Progress {
+	context.Lock()
+	defer context.Unlock()
+
+	return context._progress()
+}
+
+func (context *AptlyContext) _progress() aptly.Progress {
+	if context.progress == nil {
+		context.progress = console.NewProgress()
+		context.progress.Start()
+	}
+
+	return context.progress
+}
+
+// Downloader returns instance of current downloader
+func (context *AptlyContext) Downloader() aptly.Downloader {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.downloader == nil {
+		var downloadLimit int64
+		limitFlag := context.flags.Lookup("download-limit")
+		if limitFlag != nil {
+			downloadLimit = limitFlag.Value.Get().(int64)
+		}
+		if downloadLimit == 0 {
+			downloadLimit = context.config().DownloadLimit
+		}
+		context.downloader = http.NewDownloader(context.config().DownloadConcurrency,
+			downloadLimit*1024, context._progress())
+	}
+
+	return context.downloader
+}
+
+// DBPath builds path to database
+func (context *AptlyContext) DBPath() string {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.dbPath()
+}
+
+// DBPath builds path to database
+func (context *AptlyContext) dbPath() string {
+	return filepath.Join(context.config().RootDir, "db")
+}
+
+// Database opens and returns current instance of database
+func (context *AptlyContext) Database() (database.Storage, error) {
+	context.Lock()
+	defer context.Unlock()
+
+	return context._database()
+}
+
+func (context *AptlyContext) _database() (database.Storage, error) {
+	if context.database == nil {
+		var err error
+
+		context.database, err = database.OpenDB(context.dbPath())
+		if err != nil {
+			return nil, fmt.Errorf("can't open database: %s", err)
+		}
+	}
+
+	return context.database, nil
+}
+
+// CloseDatabase closes the db temporarily
+func (context *AptlyContext) CloseDatabase() error {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.database == nil {
+		return nil
+	}
+
+	return context.database.Close()
+}
+
+// ReOpenDatabase reopens the db after close
+func (context *AptlyContext) ReOpenDatabase() error {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.database == nil {
+		return nil
+	}
+
+	const MaxTries = 10
+	const Delay = 10 * time.Second
+
+	for try := 0; try < MaxTries; try++ {
+		err := context.database.ReOpen()
+		if err == nil || strings.Index(err.Error(), "resource temporarily unavailable") == -1 {
+			return err
+		}
+		context._progress().Printf("Unable to reopen database, sleeping %s\n", Delay)
+		<-time.After(Delay)
+	}
+
+	return fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
+}
+
+// CollectionFactory builds factory producing all kinds of collections
+func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.collectionFactory == nil {
+		db, err := context._database()
+		if err != nil {
+			Fatal(err)
+		}
+		context.collectionFactory = deb.NewCollectionFactory(db)
+	}
+
+	return context.collectionFactory
+}
+
+// PackagePool returns instance of PackagePool
+func (context *AptlyContext) PackagePool() aptly.PackagePool {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.packagePool == nil {
+		context.packagePool = files.NewPackagePool(context.config().RootDir)
+	}
+
+	return context.packagePool
+}
+
+// GetPublishedStorage returns instance of PublishedStorage
+func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
+	context.Lock()
+	defer context.Unlock()
+
+	publishedStorage, ok := context.publishedStorages[name]
+	if !ok {
+		if name == "" {
+			publishedStorage = files.NewPublishedStorage(context.config().RootDir)
+		} else if strings.HasPrefix(name, "s3:") {
+			params, ok := context.config().S3PublishRoots[name[3:]]
+			if !ok {
+				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
+			}
+
+			var err error
+			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
+				params.Region, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
+				params.EncryptionMethod, params.PlusWorkaround)
+			if err != nil {
+				Fatal(err)
+			}
+		} else if strings.HasPrefix(name, "swift:") {
+			params, ok := context.config().SwiftPublishRoots[name[6:]]
+			if !ok {
+				Fatal(fmt.Errorf("published Swift storage %v not configured", name[6:]))
+			}
+
+			var err error
+			publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password,
+				params.AuthURL, params.Tenant, params.TenantID, params.Container, params.Prefix)
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			Fatal(fmt.Errorf("unknown published storage format: %v", name))
+		}
+		context.publishedStorages[name] = publishedStorage
+	}
+
+	return publishedStorage
+}
+
+// UploadPath builds path to upload storage
+func (context *AptlyContext) UploadPath() string {
+	return filepath.Join(context.Config().RootDir, "upload")
+}
+
+// UpdateFlags sets internal copy of flags in the context
+func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) {
+	context.Lock()
+	defer context.Unlock()
+
+	context.flags = flags
+}
+
+// Flags returns current command flags
+func (context *AptlyContext) Flags() *flag.FlagSet {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.flags
+}
+
+// GlobalFlags returns flags passed to all commands
+func (context *AptlyContext) GlobalFlags() *flag.FlagSet {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.globalFlags
+}
+
+// Shutdown shuts context down
+func (context *AptlyContext) Shutdown() {
+	context.Lock()
+	defer context.Unlock()
+
+	if aptly.EnableDebug {
+		if context.fileMemProfile != nil {
+			pprof.WriteHeapProfile(context.fileMemProfile)
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+		if context.fileCPUProfile != nil {
+			pprof.StopCPUProfile()
+			context.fileCPUProfile.Close()
+			context.fileCPUProfile = nil
+		}
+		if context.fileMemProfile != nil {
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+	}
+	if context.database != nil {
+		context.database.Close()
+		context.database = nil
+	}
+	if context.downloader != nil {
+		context.downloader.Abort()
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
+// Cleanup does partial shutdown of context
+func (context *AptlyContext) Cleanup() {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.downloader != nil {
+		context.downloader.Shutdown()
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
+// NewContext initializes context with default settings
+func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
+	var err error
+
+	context := &AptlyContext{
+		flags:             flags,
+		globalFlags:       flags,
+		dependencyOptions: -1,
+		publishedStorages: map[string]aptly.PublishedStorage{},
+	}
+
+	if aptly.EnableDebug {
+		cpuprofile := flags.Lookup("cpuprofile").Value.String()
+		if cpuprofile != "" {
+			context.fileCPUProfile, err = os.Create(cpuprofile)
+			if err != nil {
+				return nil, err
+			}
+			pprof.StartCPUProfile(context.fileCPUProfile)
+		}
+
+		memprofile := flags.Lookup("memprofile").Value.String()
+		if memprofile != "" {
+			context.fileMemProfile, err = os.Create(memprofile)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		memstats := flags.Lookup("memstats").Value.String()
+		if memstats != "" {
+			interval := flags.Lookup("meminterval").Value.Get().(time.Duration)
+
+			context.fileMemStats, err = os.Create(memstats)
+			if err != nil {
+				return nil, err
+			}
+
+			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
+
+			go func() {
+				var stats runtime.MemStats
+
+				start := time.Now().UnixNano()
+
+				for {
+					runtime.ReadMemStats(&stats)
+					if context.fileMemStats != nil {
+						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
+							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
+						time.Sleep(interval)
+					} else {
+						break
+					}
+				}
+			}()
+		}
+	}
+
+	return context, nil
+}

database/leveldb.go

@@ -101,7 +101,7 @@ func (l *levelDB) Put(key []byte, value []byte) error {
 			return err
 		}
 	} else {
-		if bytes.Compare(old, value) == 0 {
+		if bytes.Equal(old, value) {
 			return nil
 		}
 	}

database/leveldb_test.go

@@ -1,8 +1,9 @@
 package database
 
 import (
-	. "launchpad.net/gocheck"
 	"testing"
+
+	. "gopkg.in/check.v1"
 )
 
 // Launch gocheck tests

deb/collections.go

@@ -2,10 +2,12 @@ package deb
 
 import (
 	"github.com/smira/aptly/database"
+	"sync"
 )
 
 // CollectionFactory is a single place to generate all desired collections
 type CollectionFactory struct {
+	*sync.Mutex
 	db             database.Storage
 	packages       *PackageCollection
 	remoteRepos    *RemoteRepoCollection
@@ -16,11 +18,14 @@ type CollectionFactory struct {
 
 // NewCollectionFactory creates new factory
 func NewCollectionFactory(db database.Storage) *CollectionFactory {
-	return &CollectionFactory{db: db}
+	return &CollectionFactory{Mutex: &sync.Mutex{}, db: db}
 }
 
 // PackageCollection returns (or creates) new PackageCollection
 func (factory *CollectionFactory) PackageCollection() *PackageCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
 	if factory.packages == nil {
 		factory.packages = NewPackageCollection(factory.db)
 	}
@@ -30,6 +35,9 @@ func (factory *CollectionFactory) PackageCollection() *PackageCollection {
 
 // RemoteRepoCollection returns (or creates) new RemoteRepoCollection
 func (factory *CollectionFactory) RemoteRepoCollection() *RemoteRepoCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
 	if factory.remoteRepos == nil {
 		factory.remoteRepos = NewRemoteRepoCollection(factory.db)
 	}
@@ -39,6 +47,9 @@ func (factory *CollectionFactory) RemoteRepoCollection() *RemoteRepoCollection {
 
 // SnapshotCollection returns (or creates) new SnapshotCollection
 func (factory *CollectionFactory) SnapshotCollection() *SnapshotCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
 	if factory.snapshots == nil {
 		factory.snapshots = NewSnapshotCollection(factory.db)
 	}
@@ -48,6 +59,9 @@ func (factory *CollectionFactory) SnapshotCollection() *SnapshotCollection {
 
 // LocalRepoCollection returns (or creates) new LocalRepoCollection
 func (factory *CollectionFactory) LocalRepoCollection() *LocalRepoCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
 	if factory.localRepos == nil {
 		factory.localRepos = NewLocalRepoCollection(factory.db)
 	}
@@ -57,9 +71,24 @@ func (factory *CollectionFactory) LocalRepoCollection() *LocalRepoCollection {
 
 // PublishedRepoCollection returns (or creates) new PublishedRepoCollection
 func (factory *CollectionFactory) PublishedRepoCollection() *PublishedRepoCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
 	if factory.publishedRepos == nil {
 		factory.publishedRepos = NewPublishedRepoCollection(factory.db)
 	}
 
 	return factory.publishedRepos
 }
+
+// Flush removes all references to collections, so that memory could be reclaimed
+func (factory *CollectionFactory) Flush() {
+	factory.Lock()
+	defer factory.Unlock()
+
+	factory.localRepos = nil
+	factory.snapshots = nil
+	factory.remoteRepos = nil
+	factory.publishedRepos = nil
+	factory.packages = nil
+}

deb/deb.go

@@ -47,7 +47,7 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 					return nil, fmt.Errorf("unable to read .tar archive: %s", err)
 				}
 
-				if tarHeader.Name == "./control" {
+				if tarHeader.Name == "./control" || tarHeader.Name == "control" {
 					reader := NewControlFileReader(untar)
 					stanza, err := reader.ReadStanza()
 					if err != nil {

deb/deb_test.go

@@ -2,9 +2,10 @@ package deb
 
 import (
 	"github.com/smira/aptly/utils"
-	. "launchpad.net/gocheck"
 	"path/filepath"
 	"runtime"
+
+	. "gopkg.in/check.v1"
 )
 
 type DebSuite struct {

deb/debian_test.go

@@ -1,8 +1,9 @@
 package deb
 
 import (
-	. "launchpad.net/gocheck"
 	"testing"
+
+	. "gopkg.in/check.v1"
 )
 
 // Launch gocheck tests

deb/format.go

@@ -11,9 +11,76 @@ import (
 type Stanza map[string]string
 
 // Canonical order of fields in stanza
-var canocialOrder = []string{"Package", "Origin", "Label", "Suite", "Version", "Installed-Size", "Priority", "Section", "Maintainer",
+// Taken from: http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/vivid/apt/vivid/view/head:/apt-pkg/tagfile.cc#L504
-	"Architecture", "Codename", "Date", "Architectures", "Components", "Description", "MD5sum", "MD5Sum", "SHA1", "SHA256",
+var (
-	"Archive", "Component"}
+	canonicalOrderRelease = []string{
+		"Origin",
+		"Label",
+		"Archive",
+		"Suite",
+		"Version",
+		"Codename",
+		"Date",
+		"Architectures",
+		"Architecture",
+		"Components",
+		"Component",
+		"Description",
+		"MD5Sum",
+		"SHA1",
+		"SHA256",
+	}
+
+	canonicalOrderBinary = []string{
+		"Package",
+		"Essential",
+		"Status",
+		"Priority",
+		"Section",
+		"Installed-Size",
+		"Maintainer",
+		"Original-Maintainer",
+		"Architecture",
+		"Source",
+		"Version",
+		"Replaces",
+		"Provides",
+		"Depends",
+		"Pre-Depends",
+		"Recommends",
+		"Suggests",
+		"Conflicts",
+		"Breaks",
+		"Conffiles",
+		"Filename",
+		"Size",
+		"MD5Sum",
+		"MD5sum",
+		"SHA1",
+		"SHA256",
+		"Description",
+	}
+
+	canonicalOrderSource = []string{
+		"Package",
+		"Source",
+		"Binary",
+		"Version",
+		"Priority",
+		"Section",
+		"Maintainer",
+		"Original-Maintainer",
+		"Build-Depends",
+		"Build-Depends-Indep",
+		"Build-Conflicts",
+		"Build-Conflicts-Indep",
+		"Architecture",
+		"Standards-Version",
+		"Format",
+		"Directory",
+		"Files",
+	}
+)
 
 // Copy returns copy of Stanza
 func (s Stanza) Copy() (result Stanza) {
@@ -41,8 +108,16 @@ func writeField(w *bufio.Writer, field, value string) (err error) {
 }
 
 // WriteTo saves stanza back to stream, modifying itself on the fly
-func (s Stanza) WriteTo(w *bufio.Writer) error {
+func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
-	for _, field := range canocialOrder {
+	canonicalOrder := canonicalOrderBinary
+	if isSource {
+		canonicalOrder = canonicalOrderSource
+	}
+	if isRelease {
+		canonicalOrder = canonicalOrderRelease
+	}
+
+	for _, field := range canonicalOrder {
 		value, ok := s[field]
 		if ok {
 			delete(s, field)

deb/format_test.go

@@ -3,8 +3,9 @@ package deb
 import (
 	"bufio"
 	"bytes"
-	. "launchpad.net/gocheck"
 	"strings"
+
+	. "gopkg.in/check.v1"
 )
 
 type ControlFileSuite struct {
@@ -107,7 +108,7 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 
 	buf := &bytes.Buffer{}
 	w := bufio.NewWriter(buf)
-	err = stanza.Copy().WriteTo(w)
+	err = stanza.Copy().WriteTo(w, false, false)
 	c.Assert(err, IsNil)
 	err = w.Flush()
 	c.Assert(err, IsNil)

deb/graph.go

@@ -0,0 +1,120 @@
+package deb
+
+import (
+	"code.google.com/p/gographviz"
+	"fmt"
+	"strings"
+)
+
+// BuildGraph generates graph contents from aptly object database
+func BuildGraph(collectionFactory *CollectionFactory) (gographviz.Interface, error) {
+	var err error
+
+	graph := gographviz.NewEscape()
+	graph.SetDir(true)
+	graph.SetName("aptly")
+
+	existingNodes := map[string]bool{}
+
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *RemoteRepo) error {
+		err := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+
+		graph.AddNode("aptly", repo.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "darkgoldenrod1",
+			"label": fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
+				repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "),
+				strings.Join(repo.Architectures, ", "), repo.NumPackages()),
+		})
+		existingNodes[repo.UUID] = true
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *LocalRepo) error {
+		err := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+
+		graph.AddNode("aptly", repo.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "mediumseagreen",
+			"label": fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
+				repo.Name, repo.Comment, repo.NumPackages()),
+		})
+		existingNodes[repo.UUID] = true
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
+		existingNodes[snapshot.UUID] = true
+		return nil
+	})
+
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
+		err := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+
+		description := snapshot.Description
+		if snapshot.SourceKind == "repo" {
+			description = "Snapshot from repo"
+		}
+
+		graph.AddNode("aptly", snapshot.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "cadetblue1",
+			"label":     fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages()),
+		})
+
+		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
+			for _, uuid := range snapshot.SourceIDs {
+				_, exists := existingNodes[uuid]
+				if exists {
+					graph.AddEdge(uuid, snapshot.UUID, true, nil)
+				}
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	collectionFactory.PublishedRepoCollection().ForEach(func(repo *PublishedRepo) error {
+		graph.AddNode("aptly", repo.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "darkolivegreen1",
+			"label": fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution,
+				strings.Join(repo.Components(), " "), strings.Join(repo.Architectures, ", ")),
+		})
+
+		for _, uuid := range repo.Sources {
+			_, exists := existingNodes[uuid]
+			if exists {
+				graph.AddEdge(uuid, repo.UUID, true, nil)
+			}
+		}
+
+		return nil
+	})
+
+	return graph, nil
+}

deb/import.go

@@ -0,0 +1,163 @@
+package deb
+
+import (
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// CollectPackageFiles walks filesystem collecting all candidates for package files
+func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, failedFiles []string, err error) {
+	for _, location := range locations {
+		info, err2 := os.Stat(location)
+		if err2 != nil {
+			reporter.Warning("Unable to process %s: %s", location, err2)
+			failedFiles = append(failedFiles, location)
+			continue
+		}
+		if info.IsDir() {
+			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
+				if err3 != nil {
+					return err3
+				}
+				if info.IsDir() {
+					return nil
+				}
+
+				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+					strings.HasSuffix(info.Name(), ".dsc") {
+					packageFiles = append(packageFiles, path)
+				}
+
+				return nil
+			})
+		} else {
+			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+				strings.HasSuffix(info.Name(), ".dsc") {
+				packageFiles = append(packageFiles, location)
+			} else {
+				reporter.Warning("Unknown file extension: %s", location)
+				failedFiles = append(failedFiles, location)
+				continue
+			}
+		}
+	}
+
+	sort.Strings(packageFiles)
+
+	return
+}
+
+// ImportPackageFiles imports files into local repository
+func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier utils.Verifier,
+	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter) (processedFiles []string, failedFiles []string, err error) {
+	if forceReplace {
+		list.PrepareIndex()
+	}
+
+	for _, file := range packageFiles {
+		var (
+			stanza Stanza
+			p      *Package
+		)
+
+		candidateProcessedFiles := []string{}
+		isSourcePackage := strings.HasSuffix(file, ".dsc")
+		isUdebPackage := strings.HasSuffix(file, ".udeb")
+
+		if isSourcePackage {
+			stanza, err = GetControlFileFromDsc(file, verifier)
+
+			if err == nil {
+				stanza["Package"] = stanza["Source"]
+				delete(stanza, "Source")
+
+				p, err = NewSourcePackageFromControlFile(stanza)
+			}
+		} else {
+			stanza, err = GetControlFileFromDeb(file)
+			if isUdebPackage {
+				p = NewUdebPackageFromControlFile(stanza)
+			} else {
+				p = NewPackageFromControlFile(stanza)
+			}
+		}
+		if err != nil {
+			reporter.Warning("Unable to read file %s: %s", file, err)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		var checksums utils.ChecksumInfo
+		checksums, err = utils.ChecksumsForFile(file)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if isSourcePackage {
+			p.UpdateFiles(append(p.Files(), PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
+		} else {
+			p.UpdateFiles([]PackageFile{{Filename: filepath.Base(file), Checksums: checksums}})
+		}
+
+		err = pool.Import(file, checksums.MD5)
+		if err != nil {
+			reporter.Warning("Unable to import file %s into pool: %s", file, err)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		candidateProcessedFiles = append(candidateProcessedFiles, file)
+
+		// go over all files, except for the last one (.dsc/.deb itself)
+		for _, f := range p.Files() {
+			if filepath.Base(f.Filename) == filepath.Base(file) {
+				continue
+			}
+			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
+			err = pool.Import(sourceFile, f.Checksums.MD5)
+			if err != nil {
+				reporter.Warning("Unable to import file %s into pool: %s", sourceFile, err)
+				failedFiles = append(failedFiles, file)
+				break
+			}
+
+			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
+		}
+		if err != nil {
+			// some files haven't been imported
+			continue
+		}
+
+		err = collection.Update(p)
+		if err != nil {
+			reporter.Warning("Unable to save package %s: %s", p, err)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		if forceReplace {
+			conflictingPackages := list.Search(Dependency{Pkg: p.Name, Version: p.Version, Relation: VersionEqual, Architecture: p.Architecture}, true)
+			for _, cp := range conflictingPackages {
+				reporter.Removed("%s removed due to conflict with package being added", cp)
+				list.Remove(cp)
+			}
+		}
+
+		err = list.Add(p)
+		if err != nil {
+			reporter.Warning("Unable to add package to repo %s: %s", p, err)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		reporter.Added("%s added", p)
+		processedFiles = append(processedFiles, candidateProcessedFiles...)
+	}
+
+	err = nil
+	return
+}

deb/index_files.go

@@ -150,7 +150,10 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 }
 
 func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
-	key := fmt.Sprintf("pi-%s-%s-%s", component, arch, udeb)
+	if arch == "source" {
+		udeb = false
+	}
+	key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb)
 	file, ok := files.indexes[key]
 	if !ok {
 		var relativePath string
@@ -180,7 +183,10 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 }
 
 func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexFile {
-	key := fmt.Sprintf("ri-%s-%s-%s", component, arch, udeb)
+	if arch == "source" {
+		udeb = false
+	}
+	key := fmt.Sprintf("ri-%s-%s-%v", component, arch, udeb)
 	file, ok := files.indexes[key]
 	if !ok {
 		var relativePath string

deb/list.go

@@ -38,6 +38,11 @@ type PackageList struct {
 	providesIndex map[string][]*Package
 }
 
+// PackageConflictError means that package can't be added to the list due to error
+type PackageConflictError struct {
+	error
+}
+
 // Verify interface
 var (
 	_ sort.Interface = &PackageList{}
@@ -90,7 +95,7 @@ func (l *PackageList) Add(p *Package) error {
 	existing, ok := l.packages[key]
 	if ok {
 		if !existing.Equals(p) {
-			return fmt.Errorf("conflict in package %s", p)
+			return &PackageConflictError{fmt.Errorf("conflict in package %s", p)}
 		}
 		return nil
 	}
@@ -205,6 +210,19 @@ func (l *PackageList) Architectures(includeSource bool) (result []string) {
 	return
 }
 
+// Strings builds list of strings with package keys
+func (l *PackageList) Strings() []string {
+	result := make([]string, l.Len())
+	i := 0
+
+	for _, p := range l.packages {
+		result[i] = string(p.Key(""))
+		i += 1
+	}
+
+	return result
+}
+
 // depSliceDeduplicate removes dups in slice of Dependencies
 func depSliceDeduplicate(s []Dependency) []Dependency {
 	l := len(s)

deb/list_test.go

@@ -2,10 +2,11 @@ package deb
 
 import (
 	"errors"
-	. "launchpad.net/gocheck"
 	"regexp"
 	"sort"
 	"strings"
+
+	. "gopkg.in/check.v1"
 )
 
 type containsChecker struct {
@@ -79,20 +80,20 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 
 	s.il = NewPackageList()
 	s.packages = []*Package{
-		&Package{Name: "lib", Version: "1.0", Architecture: "i386", Source: "lib (0.9)", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"mail-agent"}}},
+		{Name: "lib", Version: "1.0", Architecture: "i386", Source: "lib (0.9)", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"mail-agent"}}},
-		&Package{Name: "dpkg", Version: "1.7", Architecture: "i386", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
+		{Name: "dpkg", Version: "1.7", Architecture: "i386", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
-		&Package{Name: "data", Version: "1.1~bp1", Architecture: "all", Source: "app", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
+		{Name: "data", Version: "1.1~bp1", Architecture: "all", Source: "app", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "1.1~bp1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
-		&Package{Name: "mailer", Version: "3.5.8", Architecture: "i386", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		{Name: "mailer", Version: "3.5.8", Architecture: "i386", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "1.1~bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
+		{Name: "app", Version: "1.1~bp1", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
-		&Package{Name: "app", Version: "1.0", Architecture: "s390", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "1.0", Architecture: "s390", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
-		&Package{Name: "aa", Version: "2.0-1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
+		{Name: "aa", Version: "2.0-1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
-		&Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "amd64", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
+		{Name: "dpkg", Version: "1.6.1-3", Architecture: "amd64", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
-		&Package{Name: "libx", Version: "1.5", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
+		{Name: "libx", Version: "1.5", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
-		&Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "arm", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
+		{Name: "dpkg", Version: "1.6.1-3", Architecture: "arm", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
-		&Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "dpkg", Version: "1.6.1-3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
-		&Package{Name: "dpkg", Version: "1.7", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "dpkg", Version: "1.7", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
 	}
 	for _, p := range s.packages {
 		s.il.Add(p)
@@ -101,12 +102,12 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 
 	s.il2 = NewPackageList()
 	s.packages2 = []*Package{
-		&Package{Name: "mailer", Version: "3.5.8", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		{Name: "mailer", Version: "3.5.8", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
-		&Package{Name: "sendmail", Version: "1.0", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		{Name: "sendmail", Version: "1.0", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
-		&Package{Name: "app", Version: "1.1-bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "1.1-bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
-		&Package{Name: "app", Version: "1.1-bp2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "1.1-bp2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
-		&Package{Name: "app", Version: "1.2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
+		{Name: "app", Version: "1.2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
-		&Package{Name: "app", Version: "3.0", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "3.0", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
 	}
 	for _, p := range s.packages2 {
 		s.il2.Add(p)
@@ -114,10 +115,10 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 	s.il2.PrepareIndex()
 
 	s.sourcePackages = []*Package{
-		&Package{Name: "postfix", Version: "1.3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "postfix", Version: "1.3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "app", Version: "1.1~bp1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
-		&Package{Name: "aa", Version: "2.0-1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "aa", Version: "2.0-1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
-		&Package{Name: "lib", Version: "0.9", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "lib", Version: "0.9", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
 	}
 
 }
@@ -387,7 +388,7 @@ func (s *PackageListSuite) TestVerifyDependencies(c *C) {
 
 	missing, err = s.il.VerifyDependencies(0, []string{"i386", "amd64"}, s.il, nil)
 	c.Check(err, IsNil)
-	c.Check(missing, DeepEquals, []Dependency{Dependency{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "amd64"}})
+	c.Check(missing, DeepEquals, []Dependency{{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "amd64"}})
 
 	missing, err = s.il.VerifyDependencies(0, []string{"arm"}, s.il, nil)
 	c.Check(err, IsNil)
@@ -395,8 +396,8 @@ func (s *PackageListSuite) TestVerifyDependencies(c *C) {
 
 	missing, err = s.il.VerifyDependencies(DepFollowAllVariants, []string{"arm"}, s.il, nil)
 	c.Check(err, IsNil)
-	c.Check(missing, DeepEquals, []Dependency{Dependency{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "arm"},
+	c.Check(missing, DeepEquals, []Dependency{{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "arm"},
-		Dependency{Pkg: "mail-agent", Relation: VersionDontCare, Version: "", Architecture: "arm"}})
+		{Pkg: "mail-agent", Relation: VersionDontCare, Version: "", Architecture: "arm"}})
 
 	for _, p := range s.sourcePackages {
 		s.il.Add(p)
@@ -404,11 +405,11 @@ func (s *PackageListSuite) TestVerifyDependencies(c *C) {
 
 	missing, err = s.il.VerifyDependencies(DepFollowSource, []string{"i386", "amd64"}, s.il, nil)
 	c.Check(err, IsNil)
-	c.Check(missing, DeepEquals, []Dependency{Dependency{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "amd64"}})
+	c.Check(missing, DeepEquals, []Dependency{{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "amd64"}})
 
 	missing, err = s.il.VerifyDependencies(DepFollowSource, []string{"arm"}, s.il, nil)
 	c.Check(err, IsNil)
-	c.Check(missing, DeepEquals, []Dependency{Dependency{Pkg: "libx", Relation: VersionEqual, Version: "1.5", Architecture: "source"}})
+	c.Check(missing, DeepEquals, []Dependency{{Pkg: "libx", Relation: VersionEqual, Version: "1.5", Architecture: "source"}})
 
 	_, err = s.il.VerifyDependencies(0, []string{"i386", "amd64", "s390"}, s.il, nil)
 	c.Check(err, ErrorMatches, "unable to process package app_1.0_s390:.*")

deb/local.go

@@ -7,12 +7,13 @@ import (
 	"github.com/smira/aptly/database"
 	"github.com/ugorji/go/codec"
 	"log"
+	"sync"
 )
 
 // LocalRepo is a collection of packages created locally
 type LocalRepo struct {
 	// Permanent internal ID
-	UUID string
+	UUID string `json:"-"`
 	// User-assigned name
 	Name string
 	// Comment
@@ -88,6 +89,7 @@ func (repo *LocalRepo) RefKey() []byte {
 
 // LocalRepoCollection does listing, updating/adding/deleting of LocalRepos
 type LocalRepoCollection struct {
+	*sync.RWMutex
 	db   database.Storage
 	list []*LocalRepo
 }
@@ -95,7 +97,8 @@ type LocalRepoCollection struct {
 // NewLocalRepoCollection loads LocalRepos from DB and makes up collection
 func NewLocalRepoCollection(db database.Storage) *LocalRepoCollection {
 	result := &LocalRepoCollection{
-		db: db,
+		RWMutex: &sync.RWMutex{},
+		db:      db,
 	}
 
 	blobs := db.FetchByPrefix([]byte("L"))
@@ -104,7 +107,7 @@ func NewLocalRepoCollection(db database.Storage) *LocalRepoCollection {
 	for _, blob := range blobs {
 		r := &LocalRepo{}
 		if err := r.Decode(blob); err != nil {
-			log.Printf("Error decoding mirror: %s\n", err)
+			log.Printf("Error decoding repo: %s\n", err)
 		} else {
 			result.list = append(result.list, r)
 		}

deb/local_test.go

@@ -3,7 +3,8 @@ package deb
 import (
 	"errors"
 	"github.com/smira/aptly/database"
-	. "launchpad.net/gocheck"
+
+	. "gopkg.in/check.v1"
 )
 
 type LocalRepoSuite struct {

deb/package.go

@@ -1,6 +1,7 @@
 package deb
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
@@ -38,6 +39,11 @@ type Package struct {
 	collection *PackageCollection
 }
 
+// Check interface
+var (
+	_ json.Marshaler = &Package{}
+)
+
 // NewPackageFromControlFile creates Package from parsed Debian control file
 func NewPackageFromControlFile(input Stanza) *Package {
 	result := &Package{
@@ -55,12 +61,18 @@ func NewPackageFromControlFile(input Stanza) *Package {
 
 	filesize, _ := strconv.ParseInt(input["Size"], 10, 64)
 
+	md5, ok := input["MD5sum"]
+	if !ok {
+		// there are some broken repos out there with MD5 in wrong field
+		md5 = input["MD5Sum"]
+	}
+
 	result.UpdateFiles(PackageFiles{PackageFile{
 		Filename:     filepath.Base(input["Filename"]),
 		downloadPath: filepath.Dir(input["Filename"]),
 		Checksums: utils.ChecksumInfo{
 			Size:   filesize,
-			MD5:    strings.TrimSpace(input["MD5sum"]),
+			MD5:    strings.TrimSpace(md5),
 			SHA1:   strings.TrimSpace(input["SHA1"]),
 			SHA256: strings.TrimSpace(input["SHA256"]),
 		},
@@ -68,6 +80,7 @@ func NewPackageFromControlFile(input Stanza) *Package {
 
 	delete(input, "Filename")
 	delete(input, "MD5sum")
+	delete(input, "MD5Sum")
 	delete(input, "SHA1")
 	delete(input, "SHA256")
 	delete(input, "Size")
@@ -198,6 +211,16 @@ func (p *Package) String() string {
 	return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.Architecture)
 }
 
+// MarshalJSON implements json.Marshaller interface
+func (p *Package) MarshalJSON() ([]byte, error) {
+	stanza := p.Stanza()
+	stanza["FilesHash"] = fmt.Sprintf("%08x", p.FilesHash)
+	stanza["Key"] = string(p.Key(""))
+	stanza["ShortKey"] = string(p.ShortKey(""))
+
+	return json.Marshal(stanza)
+}
+
 // GetField returns fields from package
 func (p *Package) GetField(name string) string {
 	switch name {
@@ -262,7 +285,6 @@ func (p *Package) GetField(name string) string {
 	default:
 		return p.Extra()[name]
 	}
-	return ""
 }
 
 // MatchesArchitecture checks whether packages matches specified architecture
@@ -404,7 +426,9 @@ func (p *Package) Stanza() (result Stanza) {
 		result["Architecture"] = p.SourceArchitecture
 	} else {
 		result["Architecture"] = p.Architecture
-		result["Source"] = p.Source
+		if p.Source != "" {
+			result["Source"] = p.Source
+		}
 	}
 
 	if p.IsSource {

deb/package_collection_test.go

@@ -3,7 +3,8 @@ package deb
 import (
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/utils"
-	. "launchpad.net/gocheck"
+
+	. "gopkg.in/check.v1"
 )
 
 type PackageCollectionSuite struct {

deb/package_files_test.go

@@ -3,9 +3,10 @@ package deb
 import (
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/utils"
-	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
+
+	. "gopkg.in/check.v1"
 )
 
 type PackageFilesSuite struct {

deb/package_test.go

@@ -4,10 +4,11 @@ import (
 	"bytes"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/utils"
-	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
 	"regexp"
+
+	. "gopkg.in/check.v1"
 )
 
 type PackageSuite struct {
@@ -398,7 +399,7 @@ func (s *PackageSuite) TestDownloadList(c *C) {
 	list, err := p.DownloadList(packagePool)
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []PackageDownloadTask{
-		PackageDownloadTask{
+		{
 			RepoURI:         "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb",
 			DestinationPath: poolPath,
 			Checksums: utils.ChecksumInfo{Size: 5,

deb/ppa_test.go

@@ -2,7 +2,8 @@ package deb
 
 import (
 	"github.com/smira/aptly/utils"
-	. "launchpad.net/gocheck"
+
+	. "gopkg.in/check.v1"
 )
 
 type PpaSuite struct {

deb/publish.go

@@ -1,8 +1,10 @@
 package deb
 
 import (
+	"bufio"
 	"bytes"
 	"code.google.com/p/go-uuid/uuid"
+	"encoding/json"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
@@ -14,6 +16,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -56,6 +59,21 @@ type PublishedRepo struct {
 	rePublishing bool
 }
 
+// ParsePrefix splits [storage:]prefix into components
+func ParsePrefix(param string) (storage, prefix string) {
+	i := strings.LastIndex(param, ":")
+	if i != -1 {
+		storage = param[:i]
+		prefix = param[i+1:]
+		if prefix == "" {
+			prefix = "."
+		}
+	} else {
+		prefix = param
+	}
+	return
+}
+
 // walkUpTree goes from source in the tree of source snapshots/mirrors/local repos
 // gathering information about declared components and distributions
 func walkUpTree(source interface{}, collectionFactory *CollectionFactory) (rootDistributions []string, rootComponents []string) {
@@ -239,6 +257,40 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 	return result, nil
 }
 
+// MarshalJSON requires object to be "loeaded completely"
+func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
+	type sourceInfo struct {
+		Component, Name string
+	}
+
+	sources := []sourceInfo{}
+	for component, item := range p.sourceItems {
+		name := ""
+		if item.snapshot != nil {
+			name = item.snapshot.Name
+		} else if item.localRepo != nil {
+			name = item.localRepo.Name
+		} else {
+			panic("no snapshot/local repo")
+		}
+		sources = append(sources, sourceInfo{
+			Component: component,
+			Name:      name,
+		})
+	}
+
+	return json.Marshal(map[string]interface{}{
+		"Architectures": p.Architectures,
+		"Distribution":  p.Distribution,
+		"Label":         p.Label,
+		"Origin":        p.Origin,
+		"Prefix":        p.Prefix,
+		"SourceKind":    p.SourceKind,
+		"Sources":       sources,
+		"Storage":       p.Storage,
+	})
+}
+
 // String returns human-readable represenation of PublishedRepo
 func (p *PublishedRepo) String() string {
 	var sources = []string{}
@@ -471,7 +523,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			progress.InitBar(int64(list.Len()), false)
 		}
 
-		err = list.ForEach(func(pkg *Package) error {
+		list.PrepareIndex()
+
+		err = list.ForEachIndexed(func(pkg *Package) error {
 			if progress != nil {
 				progress.AddBar(1)
 			}
@@ -494,12 +548,14 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 			for _, arch := range p.Architectures {
 				if pkg.MatchesArchitecture(arch) {
-					bufWriter, err := indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
+					var bufWriter *bufio.Writer
+
+					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
 					if err != nil {
 						return err
 					}
 
-					err = pkg.Stanza().WriteTo(bufWriter)
+					err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false)
 					if err != nil {
 						return err
 					}
@@ -545,9 +601,10 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				release["Origin"] = p.GetOrigin()
 				release["Label"] = p.GetLabel()
 
-				bufWriter, err := indexes.ReleaseIndex(component, arch, udeb).BufWriter()
+				var bufWriter *bufio.Writer
+				bufWriter, err = indexes.ReleaseIndex(component, arch, udeb).BufWriter()
 
-				err = release.WriteTo(bufWriter)
+				err = release.WriteTo(bufWriter, false, true)
 				if err != nil {
 					return fmt.Errorf("unable to create Release file: %s", err)
 				}
@@ -567,6 +624,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	release := make(Stanza)
 	release["Origin"] = p.GetOrigin()
 	release["Label"] = p.GetLabel()
+	release["Suite"] = p.Distribution
 	release["Codename"] = p.Distribution
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{"source"}), " ")
@@ -589,7 +647,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		return err
 	}
 
-	err = release.WriteTo(bufWriter)
+	err = release.WriteTo(bufWriter, false, true)
 	if err != nil {
 		return fmt.Errorf("unable to create Release file: %s", err)
 	}
@@ -648,6 +706,7 @@ func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStor
 
 // PublishedRepoCollection does listing, updating/adding/deleting of PublishedRepos
 type PublishedRepoCollection struct {
+	*sync.RWMutex
 	db   database.Storage
 	list []*PublishedRepo
 }
@@ -655,7 +714,8 @@ type PublishedRepoCollection struct {
 // NewPublishedRepoCollection loads PublishedRepos from DB and makes up collection
 func NewPublishedRepoCollection(db database.Storage) *PublishedRepoCollection {
 	result := &PublishedRepoCollection{
-		db: db,
+		RWMutex: &sync.RWMutex{},
+		db:      db,
 	}
 
 	blobs := db.FetchByPrefix([]byte("U"))

deb/publish_test.go

@@ -9,9 +9,10 @@ import (
 	"github.com/smira/aptly/files"
 	"github.com/ugorji/go/codec"
 	"io/ioutil"
-	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
+
+	. "gopkg.in/check.v1"
 )
 
 type pathExistsChecker struct {
@@ -36,6 +37,9 @@ func (n *NullSigner) Init() error {
 func (n *NullSigner) SetKey(keyRef string) {
 }
 
+func (n *NullSigner) SetBatch(batch bool) {
+}
+
 func (n *NullSigner) SetKeyRing(keyring, secretKeyring string) {
 }
 

deb/reflist.go

@@ -2,6 +2,8 @@ package deb
 
 import (
 	"bytes"
+	"encoding/json"
+	"github.com/AlekSi/pointer"
 	"github.com/ugorji/go/codec"
 	"sort"
 )
@@ -92,6 +94,21 @@ func (l *PackageRefList) Has(p *Package) bool {
 	return i < len(l.Refs) && bytes.Compare(l.Refs[i], key) == 0
 }
 
+// Strings builds list of strings with package keys
+func (l *PackageRefList) Strings() []string {
+	if l == nil {
+		return []string{}
+	}
+
+	result := make([]string, l.Len())
+
+	for i := 0; i < l.Len(); i++ {
+		result[i] = string(l.Refs[i])
+	}
+
+	return result
+}
+
 // Substract returns all packages in l that are not in r
 func (l *PackageRefList) Substract(r *PackageRefList) *PackageRefList {
 	result := &PackageRefList{Refs: make([][]byte, 0, 128)}
@@ -139,6 +156,27 @@ type PackageDiff struct {
 	Left, Right *Package
 }
 
+// Check interface
+var (
+	_ json.Marshaler = PackageDiff{}
+)
+
+// MarshalJSON implements json.Marshaler interface
+func (d PackageDiff) MarshalJSON() ([]byte, error) {
+	serialized := struct {
+		Left, Right *string
+	}{}
+
+	if d.Left != nil {
+		serialized.Left = pointer.ToString(string(d.Left.Key("")))
+	}
+	if d.Right != nil {
+		serialized.Right = pointer.ToString(string(d.Right.Key("")))
+	}
+
+	return json.Marshal(serialized)
+}
+
 // PackageDiffs is a list of PackageDiff records
 type PackageDiffs []PackageDiff
 
@@ -232,8 +270,9 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle
 
 // Merge merges reflist r into current reflist. If overrideMatching, merge
 // replaces matching packages (by architecture/name) with reference from r.
-// Otherwise, all packages are saved.
+// If ignoreConflicting is set, all packages are preserved, otherwise conflciting
-func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result *PackageRefList) {
+// packages are overwritten with packages from "right" snapshot.
+func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching, ignoreConflicting bool) (result *PackageRefList) {
 	var overriddenArch, overridenName []byte
 
 	// pointer to left and right reflists
@@ -270,13 +309,23 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result
 			overridenName = nil
 			overriddenArch = nil
 		} else {
+			partsL := bytes.Split(rl, []byte(" "))
+			archL, nameL, versionL := partsL[0][1:], partsL[1], partsL[2]
+
+			partsR := bytes.Split(rr, []byte(" "))
+			archR, nameR, versionR := partsR[0][1:], partsR[1], partsR[2]
+
+			if !ignoreConflicting && bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) && bytes.Equal(versionL, versionR) {
+				// conflicting duplicates with same arch, name, version, but different file hash
+				result.Refs = append(result.Refs, r.Refs[ir])
+				il++
+				ir++
+				overridenName = nil
+				overriddenArch = nil
+				continue
+			}
+
 			if overrideMatching {
-				partsL := bytes.Split(rl, []byte(" "))
-				archL, nameL := partsL[0][1:], partsL[1]
-
-				partsR := bytes.Split(rr, []byte(" "))
-				archR, nameR := partsR[0][1:], partsR[1]
-
 				if bytes.Equal(archL, overriddenArch) && bytes.Equal(nameL, overridenName) {
 					// this package has already been overriden on the right
 					il++
@@ -314,15 +363,15 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result
 // packages and reduces it to only the latest of each package. The operations
 // are done in-place. This implements a "latest wins" approach which can be used
 // while merging two or more snapshots together.
-func FilterLatestRefs(r *PackageRefList) {
+func (l *PackageRefList) FilterLatestRefs() {
 	var (
 		lastArch, lastName, lastVer []byte
 		arch, name, ver             []byte
 		parts                       [][]byte
 	)
 
-	for i := 0; i < len(r.Refs); i++ {
+	for i := 0; i < len(l.Refs); i++ {
-		parts = bytes.Split(r.Refs[i][1:], []byte(" "))
+		parts = bytes.Split(l.Refs[i][1:], []byte(" "))
 		arch, name, ver = parts[0], parts[1], parts[2]
 
 		if bytes.Equal(arch, lastArch) && bytes.Equal(name, lastName) {
@@ -332,10 +381,10 @@ func FilterLatestRefs(r *PackageRefList) {
 			// Remove the older refs from the result
 			if vres > 0 {
 				// ver[i] > ver[i-1], remove element i-1
-				r.Refs = append(r.Refs[:i-1], r.Refs[i:]...)
+				l.Refs = append(l.Refs[:i-1], l.Refs[i:]...)
 			} else {
 				// ver[i] < ver[i-1], remove element i
-				r.Refs = append(r.Refs[:i], r.Refs[i+1:]...)
+				l.Refs = append(l.Refs[:i], l.Refs[i+1:]...)
 				arch, name, ver = lastArch, lastName, lastVer
 			}
 

deb/reflist_test.go

@@ -3,7 +3,8 @@ package deb
 import (
 	"errors"
 	"github.com/smira/aptly/database"
-	. "launchpad.net/gocheck"
+
+	. "gopkg.in/check.v1"
 )
 
 type PackageRefListSuite struct {
@@ -168,13 +169,13 @@ func (s *PackageRefListSuite) TestDiff(c *C) {
 	coll := NewPackageCollection(db)
 
 	packages := []*Package{
-		&Package{Name: "lib", Version: "1.0", Architecture: "i386"},      //0
+		{Name: "lib", Version: "1.0", Architecture: "i386"},      //0
-		&Package{Name: "dpkg", Version: "1.7", Architecture: "i386"},     //1
+		{Name: "dpkg", Version: "1.7", Architecture: "i386"},     //1
-		&Package{Name: "data", Version: "1.1~bp1", Architecture: "all"},  //2
+		{Name: "data", Version: "1.1~bp1", Architecture: "all"},  //2
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //3
+		{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //3
-		&Package{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //4
+		{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //4
-		&Package{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5
+		{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5
-		&Package{Name: "xyz", Version: "3.0", Architecture: "sparc"},     //6
+		{Name: "xyz", Version: "3.0", Architecture: "sparc"},     //6
 	}
 
 	for _, p := range packages {
@@ -240,17 +241,20 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	coll := NewPackageCollection(db)
 
 	packages := []*Package{
-		&Package{Name: "lib", Version: "1.0", Architecture: "i386"},      //0
+		{Name: "lib", Version: "1.0", Architecture: "i386"},                      //0
-		&Package{Name: "dpkg", Version: "1.7", Architecture: "i386"},     //1
+		{Name: "dpkg", Version: "1.7", Architecture: "i386"},                     //1
-		&Package{Name: "data", Version: "1.1~bp1", Architecture: "all"},  //2
+		{Name: "data", Version: "1.1~bp1", Architecture: "all"},                  //2
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //3
+		{Name: "app", Version: "1.1~bp1", Architecture: "i386"},                  //3
-		&Package{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //4
+		{Name: "app", Version: "1.1~bp2", Architecture: "i386"},                  //4
-		&Package{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5
+		{Name: "app", Version: "1.1~bp2", Architecture: "amd64"},                 //5
-		&Package{Name: "dpkg", Version: "1.0", Architecture: "i386"},     //6
+		{Name: "dpkg", Version: "1.0", Architecture: "i386"},                     //6
-		&Package{Name: "xyz", Version: "1.0", Architecture: "sparc"},     //7
+		{Name: "xyz", Version: "1.0", Architecture: "sparc"},                     //7
+		{Name: "dpkg", Version: "1.0", Architecture: "i386", FilesHash: 0x34445}, //8
+		{Name: "app", Version: "1.1~bp2", Architecture: "i386", FilesHash: 0x44}, //9
 	}
 
 	for _, p := range packages {
+		p.V06Plus = true
 		coll.Update(p)
 	}
 
@@ -268,35 +272,67 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	listB.Add(packages[5])
 	listB.Add(packages[6])
 
+	listC := NewPackageList()
+	listC.Add(packages[0])
+	listC.Add(packages[8])
+	listC.Add(packages[9])
+
 	reflistA := NewPackageRefListFromPackageList(listA)
 	reflistB := NewPackageRefListFromPackageList(listB)
+	reflistC := NewPackageRefListFromPackageList(listC)
 
-	mergeAB := reflistA.Merge(reflistB, true)
+	mergeAB := reflistA.Merge(reflistB, true, false)
-	mergeBA := reflistB.Merge(reflistA, true)
+	mergeBA := reflistB.Merge(reflistA, true, false)
+	mergeAC := reflistA.Merge(reflistC, true, false)
+	mergeBC := reflistB.Merge(reflistC, true, false)
+	mergeCB := reflistC.Merge(reflistB, true, false)
 
 	c.Check(toStrSlice(mergeAB), DeepEquals,
-		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 lib 1.0", "Psparc xyz 1.0"})
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 dpkg 1.0 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
 	c.Check(toStrSlice(mergeBA), DeepEquals,
-		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"})
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp1 00000000", "Pi386 dpkg 1.7 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
+	c.Check(toStrSlice(mergeAC), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
+	c.Check(toStrSlice(mergeBC), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000"})
+	c.Check(toStrSlice(mergeCB), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 dpkg 1.0 00000000", "Pi386 lib 1.0 00000000"})
 
-	mergeABall := reflistA.Merge(reflistB, false)
+	mergeABall := reflistA.Merge(reflistB, false, false)
-	mergeBAall := reflistB.Merge(reflistA, false)
+	mergeBAall := reflistB.Merge(reflistA, false, false)
+	mergeACall := reflistA.Merge(reflistC, false, false)
+	mergeBCall := reflistB.Merge(reflistC, false, false)
+	mergeCBall := reflistC.Merge(reflistB, false, false)
 
 	c.Check(mergeABall, DeepEquals, mergeBAall)
 	c.Check(toStrSlice(mergeBAall), DeepEquals,
-		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"})
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp1 00000000", "Pi386 app 1.1~bp2 00000000",
+			"Pi386 dpkg 1.0 00000000", "Pi386 dpkg 1.7 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
+
+	c.Check(mergeBCall, Not(DeepEquals), mergeCBall)
+	c.Check(toStrSlice(mergeACall), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pi386 app 1.1~bp1 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445",
+			"Pi386 dpkg 1.7 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
+	c.Check(toStrSlice(mergeBCall), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445",
+			"Pi386 lib 1.0 00000000"})
+
+	mergeBCwithConflicts := reflistB.Merge(reflistC, false, true)
+	c.Check(toStrSlice(mergeBCwithConflicts), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044",
+			"Pi386 dpkg 1.0 00000000", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000"})
 }
 
 func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) {
 	packages := []*Package{
-		&Package{Name: "lib", Version: "1.0", Architecture: "i386"},
+		{Name: "lib", Version: "1.0", Architecture: "i386"},
-		&Package{Name: "lib", Version: "1.2~bp1", Architecture: "i386"},
+		{Name: "lib", Version: "1.2~bp1", Architecture: "i386"},
-		&Package{Name: "lib", Version: "1.2", Architecture: "i386"},
+		{Name: "lib", Version: "1.2", Architecture: "i386"},
-		&Package{Name: "dpkg", Version: "1.2", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.2", Architecture: "i386"},
-		&Package{Name: "dpkg", Version: "1.3", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.3", Architecture: "i386"},
-		&Package{Name: "dpkg", Version: "1.3~bp2", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.3~bp2", Architecture: "i386"},
-		&Package{Name: "dpkg", Version: "1.5", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.5", Architecture: "i386"},
-		&Package{Name: "dpkg", Version: "1.6", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.6", Architecture: "i386"},
 	}
 
 	rl := NewPackageList()
@@ -310,7 +346,7 @@ func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) {
 	rl.Add(packages[7])
 
 	result := NewPackageRefListFromPackageList(rl)
-	FilterLatestRefs(result)
+	result.FilterLatestRefs()
 
 	c.Check(toStrSlice(result), DeepEquals,
 		[]string{"Pi386 dpkg 1.6", "Pi386 lib 1.2"})

deb/remote.go

@@ -14,8 +14,10 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 )
@@ -56,6 +58,8 @@ type RemoteRepo struct {
 	Filter string
 	// FilterWithDeps to include dependencies from filter query
 	FilterWithDeps bool
+	// SkipComponentCheck skips component list verification
+	SkipComponentCheck bool
 	// Status marks state of repository (being updated, no action)
 	Status int
 	// WorkerPID is PID of the process modifying the mirror (if any)
@@ -307,6 +311,9 @@ ok:
 
 	if !repo.IsFlat() {
 		architectures := strings.Split(stanza["Architectures"], " ")
+		sort.Strings(architectures)
+		// "source" architecture is never present, despite Release file claims
+		architectures = utils.StrSlicesSubstract(architectures, []string{"source"})
 		if len(repo.Architectures) == 0 {
 			repo.Architectures = architectures
 		} else {
@@ -318,14 +325,19 @@ ok:
 		}
 
 		components := strings.Split(stanza["Components"], " ")
-		for i := range components {
+		if strings.Contains(repo.Distribution, "/") {
-			components[i] = path.Base(components[i])
+			distributionLast := path.Base(repo.Distribution) + "/"
+			for i := range components {
+				if strings.HasPrefix(components[i], distributionLast) {
+					components[i] = components[i][len(distributionLast):]
+				}
+			}
 		}
 		if len(repo.Components) == 0 {
 			repo.Components = components
-		} else {
+		} else if !repo.SkipComponentCheck {
 			err = utils.StringsIsSubset(repo.Components, components,
-				fmt.Sprintf("component %%s not available in repo %s", repo))
+				fmt.Sprintf("component %%s not available in repo %s, use -force-components to override", repo))
 			if err != nil {
 				return err
 			}
@@ -380,6 +392,8 @@ ok:
 		return err
 	}
 
+	delete(stanza, "SHA512")
+
 	repo.Meta = stanza
 
 	return nil
@@ -454,7 +468,11 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 			}
 			err = repo.packageList.Add(p)
 			if err != nil {
-				return err
+				if _, ok := err.(*PackageConflictError); ok {
+					progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
+				} else {
+					return err
+				}
 			}
 
 			err = collectionFactory.PackageCollection().Update(p)
@@ -593,6 +611,7 @@ func (repo *RemoteRepo) RefKey() []byte {
 
 // RemoteRepoCollection does listing, updating/adding/deleting of RemoteRepos
 type RemoteRepoCollection struct {
+	*sync.RWMutex
 	db   database.Storage
 	list []*RemoteRepo
 }
@@ -600,7 +619,8 @@ type RemoteRepoCollection struct {
 // NewRemoteRepoCollection loads RemoteRepos from DB and makes up collection
 func NewRemoteRepoCollection(db database.Storage) *RemoteRepoCollection {
 	result := &RemoteRepoCollection{
-		db: db,
+		RWMutex: &sync.RWMutex{},
+		db:      db,
 	}
 
 	blobs := db.FetchByPrefix([]byte("R"))

deb/remote_test.go

@@ -10,9 +10,10 @@ import (
 	"github.com/smira/aptly/utils"
 	"io"
 	"io/ioutil"
-	. "launchpad.net/gocheck"
 	"os"
 	"sort"
+
+	. "gopkg.in/check.v1"
 )
 
 type NullVerifier struct {
@@ -192,7 +193,7 @@ func (s *RemoteRepoSuite) TestFetch(c *C) {
 
 func (s *RemoteRepoSuite) TestFetchNullVerifier1(c *C) {
 	downloader := http.NewFakeDownloader()
-	downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/InRelease", errors.New("404"))
+	downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/InRelease", &http.HTTPError{Code: 404})
 	downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release.gpg", "GPG")
 
@@ -252,8 +253,8 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, IsNil)
 
-	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", errors.New("HTTP 404"))
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.HTTPError{Code: 404})
-	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", errors.New("HTTP 404"))
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.HTTPError{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
 	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false)
@@ -281,11 +282,11 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, IsNil)
 
-	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", errors.New("HTTP 404"))
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.HTTPError{Code: 404})
-	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", errors.New("HTTP 404"))
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.HTTPError{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
-	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", errors.New("HTTP 404"))
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", &http.HTTPError{Code: 404})
-	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", errors.New("HTTP 404"))
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.HTTPError{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
 	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false)
@@ -322,8 +323,8 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	downloader := http.NewFakeDownloader()
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
-	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", errors.New("HTTP 404"))
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.HTTPError{Code: 404})
-	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", errors.New("HTTP 404"))
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.HTTPError{Code: 404})
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
 
 	err := s.flat.Fetch(downloader, nil)
@@ -352,11 +353,11 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 
 	downloader := http.NewFakeDownloader()
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
-	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", errors.New("HTTP 404"))
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.HTTPError{Code: 404})
-	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", errors.New("HTTP 404"))
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.HTTPError{Code: 404})
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
-	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", errors.New("HTTP 404"))
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", &http.HTTPError{Code: 404})
-	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", errors.New("HTTP 404"))
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", &http.HTTPError{Code: 404})
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
 
 	err := s.flat.Fetch(downloader, nil)

deb/snapshot.go

@@ -9,22 +9,24 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/ugorji/go/codec"
 	"log"
+	"sort"
 	"strings"
+	"sync"
 	"time"
 )
 
 // Snapshot is immutable state of repository: list of packages
 type Snapshot struct {
 	// Persisten internal ID
-	UUID string
+	UUID string `json:"-"`
 	// Human-readable name
 	Name string
 	// Date of creation
 	CreatedAt time.Time
 
 	// Source: kind + ID
-	SourceKind string
+	SourceKind string   `json:"-"`
-	SourceIDs  []string
+	SourceIDs  []string `json:"-"`
 	// Description of how snapshot was created
 	Description string
 
@@ -131,12 +133,12 @@ func (s *Snapshot) Decode(input []byte) error {
 		if strings.HasPrefix(err.Error(), "codec.decoder: readContainerLen: Unrecognized descriptor byte: hex: 80") {
 			// probably it is broken DB from go < 1.2, try decoding w/o time.Time
 			var snapshot11 struct {
-				UUID string
+				UUID      string
-				Name string
+				Name      string
 				CreatedAt []byte
 
-				SourceKind string
+				SourceKind  string
-				SourceIDs  []string
+				SourceIDs   []string
 				Description string
 			}
 
@@ -160,6 +162,7 @@ func (s *Snapshot) Decode(input []byte) error {
 
 // SnapshotCollection does listing, updating/adding/deleting of Snapshots
 type SnapshotCollection struct {
+	*sync.RWMutex
 	db   database.Storage
 	list []*Snapshot
 }
@@ -167,7 +170,8 @@ type SnapshotCollection struct {
 // NewSnapshotCollection loads Snapshots from DB and makes up collection
 func NewSnapshotCollection(db database.Storage) *SnapshotCollection {
 	result := &SnapshotCollection{
-		db: db,
+		RWMutex: &sync.RWMutex{},
+		db:      db,
 	}
 
 	blobs := db.FetchByPrefix([]byte("S"))
@@ -293,6 +297,23 @@ func (collection *SnapshotCollection) ForEach(handler func(*Snapshot) error) err
 	return err
 }
 
+// ForEachSorted runs method for each snapshot following some sort order
+func (collection *SnapshotCollection) ForEachSorted(sortMethod string, handler func(*Snapshot) error) error {
+	sorter, err := newSnapshotSorter(sortMethod, collection)
+	if err != nil {
+		return err
+	}
+
+	for _, i := range sorter.list {
+		err = handler(collection.list[i])
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // Len returns number of snapshots in collection
 // ForEach runs method for each snapshot
 func (collection *SnapshotCollection) Len() int {
@@ -324,3 +345,55 @@ func (collection *SnapshotCollection) Drop(snapshot *Snapshot) error {
 
 	return collection.db.Delete(snapshot.RefKey())
 }
+
+// Snapshot sorting methods
+const (
+	SortName = iota
+	SortTime
+)
+
+type snapshotSorter struct {
+	list       []int
+	collection *SnapshotCollection
+	sortMethod int
+}
+
+func newSnapshotSorter(sortMethod string, collection *SnapshotCollection) (*snapshotSorter, error) {
+	s := &snapshotSorter{collection: collection}
+
+	switch sortMethod {
+	case "time", "Time":
+		s.sortMethod = SortTime
+	case "name", "Name":
+		s.sortMethod = SortName
+	default:
+		return nil, fmt.Errorf("sorting method \"%s\" unknown", sortMethod)
+	}
+
+	s.list = make([]int, len(collection.list))
+	for i := range s.list {
+		s.list[i] = i
+	}
+
+	sort.Sort(s)
+
+	return s, nil
+}
+
+func (s *snapshotSorter) Swap(i, j int) {
+	s.list[i], s.list[j] = s.list[j], s.list[i]
+}
+
+func (s *snapshotSorter) Less(i, j int) bool {
+	switch s.sortMethod {
+	case SortName:
+		return s.collection.list[s.list[i]].Name < s.collection.list[s.list[j]].Name
+	case SortTime:
+		return s.collection.list[s.list[i]].CreatedAt.Before(s.collection.list[s.list[j]].CreatedAt)
+	}
+	panic("unknown sort method")
+}
+
+func (s *snapshotSorter) Len() int {
+	return len(s.list)
+}

deb/snapshot_test.go

@@ -3,7 +3,8 @@ package deb
 import (
 	"errors"
 	"github.com/smira/aptly/database"
-	. "launchpad.net/gocheck"
+
+	. "gopkg.in/check.v1"
 )
 
 type SnapshotSuite struct {

deb/version_test.go

@@ -1,7 +1,7 @@
 package deb
 
 import (
-	. "launchpad.net/gocheck"
+	. "gopkg.in/check.v1"
 )
 
 type VersionSuite struct {

files/files_test.go

@@ -1,8 +1,9 @@
 package files
 
 import (
-	. "launchpad.net/gocheck"
 	"testing"
+
+	. "gopkg.in/check.v1"
 )
 
 // Launch gocheck tests

files/package_pool.go

@@ -7,10 +7,12 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sync"
 )
 
 // PackagePool is deduplicated storage of package files on filesystem
 type PackagePool struct {
+	sync.Mutex
 	rootPath string
 }
 
@@ -50,6 +52,9 @@ func (pool *PackagePool) Path(filename string, hashMD5 string) (string, error) {
 
 // FilepathList returns file paths of all the files in the pool
 func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) {
+	pool.Lock()
+	defer pool.Unlock()
+
 	dirs, err := ioutil.ReadDir(pool.rootPath)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -93,6 +98,9 @@ func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error)
 
 // Remove deletes file in package pool returns its size
 func (pool *PackagePool) Remove(path string) (size int64, err error) {
+	pool.Lock()
+	defer pool.Unlock()
+
 	path = filepath.Join(pool.rootPath, path)
 
 	info, err := os.Stat(path)
@@ -106,6 +114,9 @@ func (pool *PackagePool) Remove(path string) (size int64, err error) {
 
 // Import copies file into package pool
 func (pool *PackagePool) Import(path string, hashMD5 string) error {
+	pool.Lock()
+	defer pool.Unlock()
+
 	source, err := os.Open(path)
 	if err != nil {
 		return err
@@ -128,12 +139,15 @@ func (pool *PackagePool) Import(path string, hashMD5 string) error {
 			// unable to stat target location?
 			return err
 		}
-		// file doesn't exist, that's ok
 	} else {
+		// target already exists
 		if targetInfo.Size() != sourceInfo.Size() {
 			// trying to overwrite file?
 			return fmt.Errorf("unable to import into pool: file %s already exists", poolPath)
 		}
+
+		// assume that target is already there
+		return nil
 	}
 
 	// create subdirs as necessary

files/package_pool_test.go

@@ -2,10 +2,11 @@ package files
 
 import (
 	"io/ioutil"
-	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
 	"runtime"
+
+	. "gopkg.in/check.v1"
 )
 
 type PackagePoolSuite struct {

files/public_test.go

@@ -2,10 +2,11 @@ package files
 
 import (
 	"io/ioutil"
-	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
 	"syscall"
+
+	. "gopkg.in/check.v1"
 )
 
 type PublishedStorageSuite struct {

http/download.go

@@ -16,6 +16,17 @@ import (
 	"strings"
 )
 
+// HTTPError is download error connected to HTTP code
+type HTTPError struct {
+	Code int
+	URL  string
+}
+
+// Error
+func (e *HTTPError) Error() string {
+	return fmt.Sprintf("HTTP code %d while fetching %s", e.Code, e.URL)
+}
+
 // Check interface
 var (
 	_ aptly.Downloader = (*downloaderImpl)(nil)
@@ -129,7 +140,19 @@ func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination s
 func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	downloader.progress.Printf("Downloading %s...\n", task.url)
 
-	resp, err := downloader.client.Get(task.url)
+	req, err := http.NewRequest("GET", task.url, nil)
+	if err != nil {
+		task.result <- fmt.Errorf("%s: %s", task.url, err)
+		return
+	}
+
+	proxyURL, _ := downloader.client.Transport.(*http.Transport).Proxy(req)
+	if proxyURL == nil && (req.URL.Scheme == "http" || req.URL.Scheme == "https") {
+		req.URL.Opaque = strings.Replace(req.URL.RequestURI(), "+", "%2b", -1)
+		req.URL.RawQuery = ""
+	}
+
+	resp, err := downloader.client.Do(req)
 	if err != nil {
 		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
@@ -139,7 +162,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	}
 
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		task.result <- fmt.Errorf("HTTP code %d while fetching %s", resp.StatusCode, task.url)
+		task.result <- &HTTPError{Code: resp.StatusCode, URL: task.url}
 		return
 	}
 
@@ -307,13 +330,16 @@ func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChe
 		}
 
 		if err != nil {
-			continue
+			if err1, ok := err.(*HTTPError); ok && (err1.Code == 404 || err1.Code == 403) {
+				continue
+			}
+			return nil, nil, err
 		}
 
 		var uncompressed io.Reader
 		uncompressed, err = method.transformation(file)
 		if err != nil {
-			continue
+			return nil, nil, err
 		}
 
 		return uncompressed, file, err

http/download_test.go

@@ -8,12 +8,13 @@ import (
 	"github.com/smira/aptly/utils"
 	"io"
 	"io/ioutil"
-	. "launchpad.net/gocheck"
 	"net"
 	"net/http"
 	"os"
 	"runtime"
 	"time"
+
+	. "gopkg.in/check.v1"
 )
 
 type DownloaderSuite struct {
@@ -210,9 +211,9 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
 	var buf []byte
 
 	expectedChecksums := map[string]utils.ChecksumInfo{
-		"file.bz2": utils.ChecksumInfo{Size: int64(len(bzipData))},
+		"file.bz2": {Size: int64(len(bzipData))},
-		"file.gz":  utils.ChecksumInfo{Size: int64(len(gzipData))},
+		"file.gz":  {Size: int64(len(gzipData))},
-		"file":     utils.ChecksumInfo{Size: int64(len(rawData))},
+		"file":     {Size: int64(len(rawData))},
 	}
 
 	// bzip2 only available
@@ -229,7 +230,7 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
 	// bzip2 not available, but gz is
 	buf = make([]byte, 4)
 	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
+	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
 	d.ExpectResponse("http://example.com/file.gz", gzipData)
 	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false)
 	c.Assert(err, IsNil)
@@ -241,8 +242,8 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
 	// bzip2 & gzip not available, but raw is
 	buf = make([]byte, 4)
 	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
+	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
-	d.ExpectError("http://example.com/file.gz", errors.New("404"))
+	d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
 	d.ExpectResponse("http://example.com/file", rawData)
 	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false)
 	c.Assert(err, IsNil)
@@ -254,14 +255,10 @@ func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
 	// gzip available, but broken
 	buf = make([]byte, 4)
 	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
+	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
 	d.ExpectResponse("http://example.com/file.gz", "x")
-	d.ExpectResponse("http://example.com/file", "recovered")
 	r, file, err = DownloadTryCompression(d, "http://example.com/file", nil, false)
-	c.Assert(err, IsNil)
+	c.Assert(err, ErrorMatches, "unexpected EOF")
-	defer file.Close()
-	io.ReadFull(r, buf)
-	c.Assert(string(buf), Equals, "reco")
 	c.Assert(d.Empty(), Equals, true)
 }
 
@@ -271,16 +268,16 @@ func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) {
 	c.Assert(err, ErrorMatches, "unexpected request.*")
 
 	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
+	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
-	d.ExpectError("http://example.com/file.gz", errors.New("404"))
+	d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
 	d.ExpectError("http://example.com/file", errors.New("403"))
 	_, _, err = DownloadTryCompression(d, "http://example.com/file", nil, false)
 	c.Assert(err, ErrorMatches, "403")
 
 	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
+	d.ExpectError("http://example.com/file.bz2", &HTTPError{Code: 404})
-	d.ExpectError("http://example.com/file.gz", errors.New("404"))
+	d.ExpectError("http://example.com/file.gz", &HTTPError{Code: 404})
 	d.ExpectResponse("http://example.com/file", rawData)
-	_, _, err = DownloadTryCompression(d, "http://example.com/file", map[string]utils.ChecksumInfo{"file": utils.ChecksumInfo{Size: 7}}, false)
+	_, _, err = DownloadTryCompression(d, "http://example.com/file", map[string]utils.ChecksumInfo{"file": {Size: 7}}, false)
 	c.Assert(err, ErrorMatches, "checksums don't match.*")
 }

http/http_test.go

@@ -1,8 +1,9 @@
 package http
 
 import (
-	. "launchpad.net/gocheck"
 	"testing"
+
+	. "gopkg.in/check.v1"
 )
 
 // Launch gocheck tests

man/aptly.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "APTLY" "1" "October 2014" "" ""
+.TH "APTLY" "1" "February 2015" "" ""
 .
 .SH "NAME"
 \fBaptly\fR \- Debian repository management tool
@@ -52,7 +52,7 @@ Configuration file is stored in JSON format (default values shown below):
     "test": {
       "region": "us\-east\-1",
       "bucket": "repo",
-      "awsAccessKeyID": ""
+      "awsAccessKeyID": "",
       "awsSecretAccessKey": "",
       "prefix": "",
       "acl": "public\-read",
@@ -60,6 +60,18 @@ Configuration file is stored in JSON format (default values shown below):
       "encryptionMethod": "",
       "plusWorkaround": false
     }
+  },
+  "SwiftPublishEndpoints": {
+    "test": {
+      "container": "repo",
+      "osname": "",
+      "password": "",
+      "prefix": "",
+      "authurl": "",
+      "tenant": "",
+      "tenantid": ""
+    }
+  }
 }
 .
 .fi
@@ -162,6 +174,35 @@ In order to publish to S3, specify endpoint as \fBs3:endpoint\-name:\fR before p
 .P
 \fBaptly publish snapshot wheezy\-main s3:test:\fR
 .
+.SH "OPENSTACK SWIFT PUBLISHING ENDPOINTS"
+aptly could be configured to publish repository directly to OpenStack Swift\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings:
+.
+.TP
+\fBcontainer\fR
+container name
+.
+.TP
+\fBprefix\fR
+(optional) do publishing under specified prefix in the container, defaults to no prefix (container root)
+.
+.TP
+\fBosname\fR, \fBpassword\fR
+(optional) OpenStack credentials to access Keystone\. If not supplied, environment variables \fBOS_USERNAME\fR and \fBOS_PASSWORD\fR are used\.
+.
+.TP
+\fBtenant\fR, \fBtenantid\fR
+(optional) OpenStack tenant name and id (in order to use v2 authentication)\.
+.
+.TP
+\fBauthurl\fR
+(optional) the full url of Keystone server (including port, and version)\. example \fBhttp://identity\.example\.com:5000/v2\.0\fR
+.
+.P
+In order to publish to Swift, specify endpoint as \fBswift:endpoint\-name:\fR before publishing prefix on the command line, e\.g\.:
+.
+.P
+\fBaptly publish snapshot jessie\-main swift:test:\fR
+.
 .SH "PACKAGE QUERY"
 Some commands accept package queries to identify list of packages to process\. Package query syntax almost matches \fBreprepro\fR query language\. Query consists of the following simple terms:
 .
@@ -315,6 +356,10 @@ filter packages in mirror
 when filtering, include dependencies of matching packages as well
 .
 .TP
+\-\fBforce\-components\fR=false
+(only with component list) skip check that requested components are listed in Release file
+.
+.TP
 \-\fBignore\-signatures\fR=false
 disable verification of Release file signatures
 .
@@ -1102,6 +1147,10 @@ $ aptly publish repo testing
 Options:
 .
 .TP
+\-\fBbatch\fR=false
+run GPG with detached tty
+.
+.TP
 \-\fBcomponent\fR=
 component name to publish (for multi\-component publishing, separate components with commas)
 .
@@ -1181,6 +1230,10 @@ $ aptly publish snapshot wheezy\-main
 Options:
 .
 .TP
+\-\fBbatch\fR=false
+run GPG with detached tty
+.
+.TP
 \-\fBcomponent\fR=
 component name to publish (for multi\-component publishing, separate components with commas)
 .
@@ -1228,7 +1281,7 @@ don\(cqt sign Release files with GPG
 \fBaptly\fR \fBpublish\fR \fBswitch\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR] \fInew\-snapshot\fR
 .
 .P
-Command switches in\-place published repository with new snapshot contents\. All publishing parameters are preserved (architecture list, distribution, component)\.
+Command switches in\-place published snapshots with new snapshot contents\. All publishing parameters are preserved (architecture list, distribution, component)\.
 .
 .P
 For multiple component repositories, flag \-component should be given with list of components to update\. Corresponding snapshots should be given in the same order, e\.g\.:
@@ -1237,7 +1290,7 @@ For multiple component repositories, flag \-component should be given with list
 .
 .nf
 
-aptly publish update \-component=main,contrib wheezy wh\-main wh\-contrib
+aptly publish switch \-component=main,contrib wheezy wh\-main wh\-contrib
 .
 .fi
 .
@@ -1250,16 +1303,23 @@ Example:
 .
 .nf
 
-$ aptly publish update wheezy ppa wheezy\-7\.5
+$ aptly publish switch wheezy ppa wheezy\-7\.5
 .
 .fi
 .
 .IP "" 0
 .
 .P
+This command would switch published repository (with one component) named ppa/wheezy (prefix ppa, dsitribution wheezy to new snapshot wheezy\-7\.5)\.
+.
+.P
 Options:
 .
 .TP
+\-\fBbatch\fR=false
+run GPG with detached tty
+.
+.TP
 \-\fBcomponent\fR=
 component names to update (for multi\-component publishing, separate components with commas)
 .
@@ -1317,6 +1377,10 @@ $ aptly publish update wheezy ppa
 Options:
 .
 .TP
+\-\fBbatch\fR=false
+run GPG with detached tty
+.
+.TP
 \-\fBforce\-overwrite\fR=false
 overwrite files in package pool in case of mismatch
 .
@@ -1363,7 +1427,7 @@ $ aptly package search \(cq$Architecture (i386), Name (% *\-dev)\(cq
 .
 .IP "" 0
 .
-.SH "SHOW DETAILS ABOUT PACKAGES MATCING QUERY"
+.SH "SHOW DETAILS ABOUT PACKAGES MATCHING QUERY"
 \fBaptly\fR \fBpackage\fR \fBshow\fR \fIpackage\-query\fR
 .
 .P
@@ -1448,6 +1512,49 @@ Example:
 .P
 $ aptly graph
 .
+.SH "SHOW CURRENT APTLY\(cqS CONFIG"
+\fBaptly\fR \fBconfig\fR \fBshow\fR
+.
+.P
+Command show displays the current aptly configuration\.
+.
+.P
+Example:
+.
+.P
+$ aptly config show
+.
+.SH "RUN APTLY TASKS"
+\fBaptly\fR \fBtask\fR \fBrun\fR \-filename=\fIfilename\fR \fB|\fR \fIcommand1\fR, \fIcommand2\fR, \fB\|\.\|\.\|\.\fR
+.
+.P
+Command helps organise multiple aptly commands in one single aptly task, running as single thread\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+  $ aptly task run
+  > repo create local
+  > repo add local pkg1
+  > publish repo local
+  > serve
+  >
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBfilename\fR=
+specifies the filename that contains the commands to run
+.
 .SH "ENVIRONMENT"
 If environment variable \fBHTTP_PROXY\fR is set \fBaptly\fR would use its value to proxy all HTTP requests\.
 .

man/aptly.1.ronn.tmpl

@@ -44,7 +44,7 @@ Configuration file is stored in JSON format (default values shown below):
         "test": {
           "region": "us-east-1",
           "bucket": "repo",
-          "awsAccessKeyID": ""
+          "awsAccessKeyID": "",
           "awsSecretAccessKey": "",
           "prefix": "",
           "acl": "public-read",
@@ -52,6 +52,18 @@ Configuration file is stored in JSON format (default values shown below):
           "encryptionMethod": "",
           "plusWorkaround": false
         }
+      },
+      "SwiftPublishEndpoints": {
+        "test": {
+          "container": "repo",
+          "osname": "",
+          "password": "",
+          "prefix": "",
+          "authurl": "",
+          "tenant": "",
+          "tenantid": ""
+        }
+      }
     }
 
 Options:
@@ -101,6 +113,9 @@ Options:
   * `S3PublishEndpoints`:
     configuration of Amazon S3 publishing endpoints (see below)
 
+  * `SwiftPublishEndpoints`:
+    configuration of OpenStack Swift publishing endpoints (see below)
+
 ## S3 PUBLISHING ENDPOINTS
 
 aptly could be configured to publish repository directly to Amazon S3. First, publishing
@@ -144,6 +159,31 @@ publishing prefix on the command line, e.g.:
 
   `aptly publish snapshot wheezy-main s3:test:`
 
+## OPENSTACK SWIFT PUBLISHING ENDPOINTS
+
+aptly could be configured to publish repository directly to OpenStack Swift. First,
+publishing endpoints should be described in aptly configuration file. Each endpoint
+has name and associated settings:
+
+   * `container`:
+     container name
+   * `prefix`:
+     (optional) do publishing under specified prefix in the container, defaults to
+     no prefix (container root)
+   * `osname`, `password`:
+     (optional) OpenStack credentials to access Keystone. If not supplied,
+     environment variables `OS_USERNAME` and `OS_PASSWORD` are used.
+   * `tenant`, `tenantid`:
+     (optional) OpenStack tenant name and id (in order to use v2 authentication).
+   * `authurl`:
+     (optional) the full url of Keystone server (including port, and version).
+     example `http://identity.example.com:5000/v2.0`
+
+In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before
+publishing prefix on the command line, e.g.:
+
+  `aptly publish snapshot jessie-main swift:test:`
+
 ## PACKAGE QUERY
 
 Some commands accept package queries to identify list of packages to process.
@@ -245,6 +285,10 @@ When specified on command line, query may have to be quoted according to shell r
 
 {{template "command" findCommand . "graph"}}
 
+{{template "command" findCommand . "config"}}
+
+{{template "command" findCommand . "task"}}
+
 ## ENVIRONMENT
 
 If environment variable `HTTP_PROXY` is set `aptly` would use its value
@@ -295,4 +339,4 @@ Options:
   * -`{{.Name}}`={{.DefValue}}:
     {{.Usage}}
 {{end}}
-{{end}}
+{{end}}

query/lex_test.go

@@ -2,7 +2,8 @@ package query
 
 import (
 	"fmt"
-	. "launchpad.net/gocheck"
+
+	. "gopkg.in/check.v1"
 )
 
 type LexerSuite struct {