Release 0.9.7.

Add link to scala sbt plugin

.goxc.json

@@ -0,0 +1,38 @@
+{
+	"AppName": "aptly",
+	"ArtifactsDest": "xc-out/",
+	"TasksExclude": [
+		"rmbin"
+	],
+    "TasksAppend": [
+        "bintray"
+    ],
+    "TaskSettings": {
+        "deb": {
+            "metadata": {
+                "maintainer": "Andrey Smirnov",
+                "maintainerEmail": "me@smira.ru",
+                "description": "Debian repository management tool"
+            },
+            "metadata-deb": {
+                "License": "MIT",
+                "Homepage": "https://www.aptly.info/",
+                "Recommends": "bzip2, graphviz, xz-utils",
+                "Depends": ""
+            },
+            "other-mapped-files": {
+                "/": "root/"
+            }
+        },
+        "bintray": {
+            "repository": "aptly",
+            "subject": "smira",
+            "package": "aptly",
+            "downloadspage": "bintray.md"
+        }
+    },
+	"Arch": "386 amd64",
+	"Os": "linux darwin freebsd",
+	"MainDirsExclude": "man,_vendor",
+	"ConfigVersion": "0.9"
+}

.travis.yml

@@ -1,23 +1,40 @@
+sudo: false
+
 language: go
 
 go:
-  - 1.2.1
+  - 1.4
-  - 1.3.1
+  - 1.5
   - tip
 
+addons:
+  apt:
+    packages:
+    - python-virtualenv
+    - graphviz
+
 env:
   global:
     - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
     - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
+    - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
 before_install:
-  - sudo apt-get update -qq
+  - virtualenv env
-  - sudo apt-get install -y python-boto
+  - . env/bin/activate
+  - pip install boto requests python-swiftclient
 install:
   - make prepare
 
-
 script: make travis
 
 matrix:
     allow_failures:
         - go: tip
+
+notifications:
+  webhooks:
+    urls:
+      - "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
+    on_success: change  # options: [always|never|change] default: always
+    on_failure: always  # options: [always|never|change] default: always
+    on_start: false     # default: false

AUTHORS

@@ -3,5 +3,22 @@ List of contributors, in chronological order:
 * Andrey Smirnov (https://github.com/smira)
 * Sebastien Binet (https://github.com/sbinet)
 * Ryan Uber (https://github.com/ryanuber)
-* Simon Aquino (https://github.com/simonaquino)
+* Simon Aquino (https://github.com/queeno)
-* Vincent Batoufflet (https://github.com/vbatoufflet)
+* Vincent Batoufflet (https://github.com/vbatoufflet)
+* Ivan Kurnosov (https://github.com/zerkms)
+* Dmitrii Kashin (https://github.com/freehck)
+* Chris Read (https://github.com/cread)
+* Rohan Garg (https://github.com/shadeslayer)
+* Russ Allbery (https://github.com/rra)
+* Sylvain Baubeau (https://github.com/lebauce)
+* Andrea Bernardo Ciddio (https://github.com/bcandrea)
+* Michael Koval (https://github.com/mkoval)
+* Alexander Guy (https://github.com/alexanderguy)
+* Sebastien Badia (https://github.com/sbadia)
+* Szymon Sobik (https://github.com/sobczyk)
+* Paul Krohn (https://github.com/paul-krohn)
+* Vincent Bernat (https://github.com/vincentbernat)
+* x539 (https://github.com/x539)
+* Phil Frost (https://github.com/bitglue)
+* Benoit Foucher (https://github.com/bentoi)
+* Geoffrey Thomas (https://github.com/geofft)

Gomfile

@@ -1,27 +1,38 @@
-gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
+gom 'github.com/AlekSi/pointer', :commit => '5f6d527dae3d678b46fbb20331ddf44e2b841943'
-gom 'code.google.com/p/go.crypto/ssh/terminal', :commit => '7aa593ce8cea'
+gom 'github.com/awalterschulze/gographviz', :commit => '20d1f693416d9be045340150094aa42035a41c9e'
-gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2'
+gom 'github.com/aws/aws-sdk-go', :commit => 'a170e9cb76475a0da7c0326a13cc2b39e9244b3b'
-gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556'
+gom 'github.com/cheggaaa/pb', :commit => '2c1b74620cc58a81ac152ee2d322e28c806d81ed'
-gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
+gom 'github.com/DisposaBoy/JsonConfigReader', :commit => '33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4'
-gom 'github.com/cheggaaa/pb', :commit => '74be7a1388046f374ac36e93d46f5d56e856f827'
+gom 'github.com/gin-gonic/gin', :commit => 'b1758d3bfa09e61ddbc1c9a627e936eec6a170de'
-gom 'github.com/vaughan0/go-ini', :commit => 'a98ad7ee00ec53921f08832bc06ecf7fd600e6a1'
+gom 'github.com/go-ini/ini', :commit => 'afbd495e5aaea13597b5e14fe514ddeaa4d76fc3'
+gom 'github.com/jlaffaye/ftp', :commit => 'fec71e62e457557fbe85cefc847a048d57815d76'
+gom 'github.com/jmespath/go-jmespath', :commit => '0b12d6b521d83fc7f755e7cfc1b1fbdd35a01a74'
+gom 'github.com/julienschmidt/httprouter', :commit => '46807412fe50aaceb73bb57061c2230fd26a1640'
 gom 'github.com/mattn/go-shellwords', :commit => 'c7ca6f94add751566a61cf2199e1de78d4c3eee4'
-gom 'github.com/mitchellh/goamz/s3', :commit => 'e7664b32019f31fd1bdf33f9e85f28722f700405'
+gom 'github.com/mkrautz/goar', :commit => '282caa8bd9daba480b51f1d5a988714913b97aad'
-gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
+gom 'github.com/mxk/go-flowrate/flowrate', :commit => 'cca7078d478f8520f85629ad7c68962d31ed7682'
+gom 'github.com/ncw/swift', :commit => '384ef27c70645e285f8bb9d02276bf654d06027e'
+gom 'github.com/smira/go-aws-auth', :commit => '0070896e9d7f4f9f2d558532b2d896ce2239992a'
+gom 'github.com/smira/go-xz', :commit => '0c531f070014e218b21f3cfca801cc992d52726d'
 gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
 gom 'github.com/smira/flag', :commit => '357ed3e599ffcbd4aeaa828e1d10da2df3ea5107'
 gom 'github.com/smira/go-ftp-protocol/protocol', :commit => '066b75c2b70dca7ae10b1b88b47534a3c31ccfaa'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => 'e2fa4e6ac1cc41a73bc9fd467878ecbf65df5cc3'
+gom 'github.com/smira/go-uuid/uuid', :commit => 'ed3ca8a15a931b141440a7e98e4f716eec255f7d'
+gom 'github.com/smira/lzma', :commit => '7f0af6269940baa2c938fabe73e0d7ba41205683'
+gom 'github.com/golang/snappy', :commit => '723cc1e459b8eea2dea4583200fd60757d40097a'
+gom 'github.com/syndtr/goleveldb/leveldb', :commit => '917f41c560270110ceb73c5b38be2a9127387071'
 gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
+gom 'github.com/vaughan0/go-ini', :commit => 'a98ad7ee00ec53921f08832bc06ecf7fd600e6a1'
 gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
+gom 'golang.org/x/crypto/ssh/terminal', :commit => 'a7ead6ddf06233883deca151dffaef2effbf498f'
 
 group :test do
-    gom 'launchpad.net/gocheck'
+    gom 'gopkg.in/check.v1'
 end
 
 group :development do
    gom 'github.com/golang/lint/golint'
    gom 'github.com/mattn/goveralls'
    gom 'github.com/axw/gocov/gocov'
-   gom 'code.google.com/p/go.tools/cmd/cover'
+   gom 'golang.org/x/tools/cmd/cover'
 end

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2013-2014 Andrey Smirnov. All rights reserved.
+Copyright 2013-2015 aptly authors. All rights reserved.
 
 MIT License
 

Makefile

@@ -1,6 +1,6 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-PACKAGES=database deb files http query s3 utils
+PACKAGES=context database deb files http query swift s3 utils
-ALL_PACKAGES=aptly cmd console database deb files http query s3 utils
+ALL_PACKAGES=api aptly context cmd console database deb files http query swift s3 utils
 BINPATH=$(abspath ./_vendor/bin)
 GOM_ENVIRONMENT=-test
 PYTHON?=python
@@ -36,7 +36,7 @@ coverage: coverage.out
 	rm -f coverage.out
 
 check:
-	$(GOM) exec go tool vet -all=true -shadow=true $(ALL_PACKAGES:%=./%)
+	$(GOM) exec go tool vet -all=true $(ALL_PACKAGES:%=./%)
 	$(GOM) exec golint $(ALL_PACKAGES:%=./%)
 
 install:
@@ -59,17 +59,6 @@ mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-package:
-	rm -rf root/
-	mkdir -p root/usr/bin/ root/usr/share/man/man1/ root/etc/bash_completion.d
-	cp $(BINPATH)/aptly root/usr/bin
-	cp man/aptly.1 root/usr/share/man/man1
-	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
-	gzip root/usr/share/man/man1/aptly.1
-	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
-	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 -C root/ .
-	mv aptly_$(VERSION)_*.deb ~
-
 src-package:
 	rm -rf aptly-$(VERSION)
 	mkdir -p aptly-$(VERSION)/src/github.com/smira/aptly/
@@ -77,8 +66,18 @@ src-package:
 	cd aptly-$(VERSION)/src/github.com/smira/aptly && gom -production install
 	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . \( -name .git -o -name .bzr -o -name .hg \) -print | xargs rm -rf
 	rm -rf aptly-$(VERSION)/src/github.com/smira/aptly/_vendor/{pkg,bin}
+	mkdir -p aptly-$(VERSION)/bash_completion.d
+	(cd aptly-$(VERSION)/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/$(VERSION)/aptly)
 	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
 	rm -rf aptly-$(VERSION)
 	curl -T aptly-$(VERSION)-src.tar.bz2 -usmira:$(BINTRAY_KEY) https://api.bintray.com/content/smira/aptly/aptly/$(VERSION)/$(VERSION)/aptly-$(VERSION)-src.tar.bz2
 
+goxc:
+	rm -rf root/
+	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d
+	cp man/aptly.1 root/usr/share/man/man1
+	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
+	gzip root/usr/share/man/man1/aptly.1
+	gom exec goxc -pv=$(VERSION) -max-processors=4 $(GOXC_OPTS)
+
 .PHONY: coverage.out

README.rst

@@ -8,8 +8,11 @@ aptly
 .. image:: https://coveralls.io/repos/smira/aptly/badge.png?branch=HEAD
     :target: https://coveralls.io/r/smira/aptly?branch=HEAD
 
-.. image:: http://gobuild.io/badge/github.com/smira/aptly/download.png
+.. image:: https://badges.gitter.im/Join Chat.svg
-    :target: http://gobuild.io/github.com/smira/aptly
+    :target: https://gitter.im/smira/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+
+.. image:: http://goreportcard.com/badge/gojp/goreportcard
+    :target: http://goreportcard.com/report/gojp/goreportcard
 
 Aptly is a swiss army knife for Debian repository management.
 
@@ -28,6 +31,7 @@ Aptly features: ("+" means planned features)
 * merge two or more snapshots into one
 * filter repository by search query, pulling dependencies when required
 * publish self-made packages as Debian repositories
+* REST API for remote access
 * mirror repositories "as-is" (without resigning with user's key) (+)
 * support for yum repositories (+)
 
@@ -44,8 +48,7 @@ To install aptly on Debian/Ubuntu, add new repository to /etc/apt/sources.list::
 
 And import key that is used to sign the release::
 
-    $ gpg --keyserver keys.gnupg.net --recv-keys 2A194991
+    $ apt-key adv --keyserver keys.gnupg.net --recv-keys 9E3E53F19C7DE460
-    $ gpg -a --export 2A194991 | sudo apt-key add -
 
 After that you can install aptly as any other software package::
 
@@ -55,9 +58,13 @@ After that you can install aptly as any other software package::
 Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
 Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
 
+If you would like to use nightly builds (unstable), please use following repository::
+
+    deb http://repo.aptly.info/ nightly main
+
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.2+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.4+ required)::
 
     go get -u github.com/mattn/gom
     mkdir -p $GOPATH/src/github.com/smira/aptly
@@ -71,4 +78,34 @@ should work as well, but might fail or produce different result (if external lib
 
 If you don't have Go installed (or older version), you can easily install Go using `gvm <https://github.com/moovweb/gvm/>`_.
 
+Integrations
+------------
 
+Vagrant:
+
+-   `Vagrant configuration <https://github.com/sepulworld/aptly-vagrant>`_ by
+    Zane Williamson, allowing to bring two virtual servers, one with aptly installed
+    and another one set up to install packages from repository published by aptly
+
+Docker:
+
+-    `Docker container <https://github.com/mikepurvis/aptly-docker>`_ with aptly inside by Mike Purvis
+-    `Docker container <https://github.com/bryanhong/docker-aptly>`_ with aptly and nginx by Bryan Hong
+
+With configuration management systems:
+
+-   `Chef cookbook <https://github.com/hw-cookbooks/aptly>`_ by Aaron Baer
+    (Heavy Water Operations, LLC)
+-   `Puppet module <https://github.com/alphagov/puppet-aptly>`_ by
+    Government Digital Services
+-   `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by
+    Forrest Alvarez and Brian Jackson
+-   `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine
+
+CLI for aptly API:
+
+-   `Ruby aptly CLI/library <https://github.com/sepulworld/aptly_cli>`_ by Zane Williamson
+
+Scala sbt:
+
+-   `sbt aptly plugin <https://github.com/amalakar/sbt-aptly>`_ by Arup Malakar

api/api.go

@@ -0,0 +1,159 @@
+// Package api provides implementation of aptly REST API
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"sort"
+	"time"
+)
+
+// Lock order acquisition (canonical):
+//  1. RemoteRepoCollection
+//  2. LocalRepoCollection
+//  3. SnapshotCollection
+//  4. PublishedRepoCollection
+
+// GET /api/version
+func apiVersion(c *gin.Context) {
+	c.JSON(200, gin.H{"Version": aptly.Version})
+}
+
+const (
+	ACQUIREDB = iota
+	RELEASEDB
+)
+
+// Flushes all collections which cache in-memory objects
+func flushColections() {
+	// lock everything to eliminate in-progress calls
+	r := context.CollectionFactory().RemoteRepoCollection()
+	r.Lock()
+	defer r.Unlock()
+
+	l := context.CollectionFactory().LocalRepoCollection()
+	l.Lock()
+	defer l.Unlock()
+
+	s := context.CollectionFactory().SnapshotCollection()
+	s.Lock()
+	defer s.Unlock()
+
+	p := context.CollectionFactory().PublishedRepoCollection()
+	p.Lock()
+	defer p.Unlock()
+
+	// all collections locked, flush them
+	context.CollectionFactory().Flush()
+}
+
+// Periodically flushes CollectionFactory to free up memory used by
+// collections, flushing caches. If the two channels are provided,
+// they are used to acquire and release the database.
+//
+// Should be run in goroutine!
+func cacheFlusher(requests chan int, acks chan error) {
+	ticker := time.Tick(15 * time.Minute)
+
+	for {
+		<-ticker
+
+		// if aptly API runs in -no-lock mode,
+		// caches are flushed when DB is closed anyway, no need
+		// to flush them here
+		if requests == nil {
+			flushColections()
+		}
+	}
+}
+
+// Acquire database lock and release it when not needed anymore. Two
+// channels must be provided. The first one is to receive requests to
+// acquire/release the database and the second one is to send acks.
+//
+// Should be run in a goroutine!
+func acquireDatabase(requests chan int, acks chan error) {
+	clients := 0
+	for {
+		request := <-requests
+		switch request {
+		case ACQUIREDB:
+			if clients == 0 {
+				acks <- context.ReOpenDatabase()
+			} else {
+				acks <- nil
+			}
+			clients++
+		case RELEASEDB:
+			clients--
+			if clients == 0 {
+				flushColections()
+				acks <- context.CloseDatabase()
+			} else {
+				acks <- nil
+			}
+		}
+	}
+}
+
+// Common piece of code to show list of packages,
+// with searching & details if requested
+func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
+	result := []*deb.Package{}
+
+	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), nil)
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	queryS := c.Request.URL.Query().Get("q")
+	if queryS != "" {
+		q, err := query.Parse(c.Request.URL.Query().Get("q"))
+		if err != nil {
+			c.Fail(400, err)
+			return
+		}
+
+		withDeps := c.Request.URL.Query().Get("withDeps") == "1"
+		architecturesList := []string{}
+
+		if withDeps {
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = list.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				return
+			}
+		}
+
+		list.PrepareIndex()
+
+		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
+			nil, context.DependencyOptions(), architecturesList)
+		if err != nil {
+			c.Fail(500, fmt.Errorf("unable to search: %s", err))
+			return
+		}
+	}
+
+	if c.Request.URL.Query().Get("format") == "details" {
+		list.ForEach(func(p *deb.Package) error {
+			result = append(result, p)
+			return nil
+		})
+
+		c.JSON(200, result)
+	} else {
+		c.JSON(200, list.Strings())
+	}
+}

api/files.go

@@ -0,0 +1,185 @@
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func verifyPath(path string) bool {
+	path = filepath.Clean(path)
+	for _, part := range strings.Split(path, string(filepath.Separator)) {
+		if part == ".." || part == "." {
+			return false
+		}
+	}
+
+	return true
+
+}
+
+func verifyDir(c *gin.Context) bool {
+	if !verifyPath(c.Params.ByName("dir")) {
+		c.Fail(400, fmt.Errorf("wrong dir"))
+		return false
+	}
+
+	return true
+}
+
+// GET /files
+func apiFilesListDirs(c *gin.Context) {
+	list := []string{}
+
+	err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == context.UploadPath() {
+			return nil
+		}
+
+		if info.IsDir() {
+			list = append(list, filepath.Base(path))
+			return filepath.SkipDir
+		}
+
+		return nil
+	})
+
+	if err != nil && !os.IsNotExist(err) {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// POST /files/:dir/
+func apiFilesUpload(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	err := os.MkdirAll(path, 0777)
+
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	stored := []string{}
+
+	for _, files := range c.Request.MultipartForm.File {
+		for _, file := range files {
+			src, err := file.Open()
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+			defer src.Close()
+
+			destPath := filepath.Join(path, filepath.Base(file.Filename))
+			dst, err := os.Create(destPath)
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+			defer dst.Close()
+
+			_, err = io.Copy(dst, src)
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+
+			stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
+		}
+	}
+
+	c.JSON(200, stored)
+
+}
+
+// GET /files/:dir
+func apiFilesListFiles(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	list := []string{}
+	root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == root {
+			return nil
+		}
+
+		list = append(list, filepath.Base(path))
+
+		return nil
+	})
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			c.Fail(404, err)
+		} else {
+			c.Fail(500, err)
+		}
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// DELETE /files/:dir
+func apiFilesDeleteDir(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// DELETE /files/:dir/:name
+func apiFilesDeleteFile(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	if !verifyPath(c.Params.ByName("name")) {
+		c.Fail(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
+	if err != nil {
+		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
+			c.Fail(500, err)
+			return
+		}
+	}
+
+	c.JSON(200, gin.H{})
+}

api/graph.go

@@ -0,0 +1,75 @@
+package api
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/deb"
+	"io"
+	"mime"
+	"os"
+	"os/exec"
+)
+
+// GET /api/graph.:ext
+func apiGraph(c *gin.Context) {
+	var (
+		err    error
+		output []byte
+	)
+
+	ext := c.Params.ByName("ext")
+
+	factory := context.CollectionFactory()
+
+	factory.RemoteRepoCollection().RLock()
+	defer factory.RemoteRepoCollection().RUnlock()
+	factory.LocalRepoCollection().RLock()
+	defer factory.LocalRepoCollection().RUnlock()
+	factory.SnapshotCollection().RLock()
+	defer factory.SnapshotCollection().RUnlock()
+	factory.PublishedRepoCollection().RLock()
+	defer factory.PublishedRepoCollection().RUnlock()
+
+	graph, err := deb.BuildGraph(factory)
+	if err != nil {
+		c.JSON(500, err)
+		return
+	}
+
+	buf := bytes.NewBufferString(graph.String())
+
+	command := exec.Command("dot", "-T"+ext)
+	command.Stderr = os.Stderr
+
+	stdin, err := command.StdinPipe()
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	_, err = io.Copy(stdin, buf)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	err = stdin.Close()
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	output, err = command.Output()
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		return
+	}
+
+	mimeType := mime.TypeByExtension("." + ext)
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+
+	c.Data(200, mimeType, output)
+}

api/packages.go

@@ -0,0 +1,16 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/packages/:key
+func apiPackagesShow(c *gin.Context) {
+	p, err := context.CollectionFactory().PackageCollection().ByKey([]byte(c.Params.ByName("key")))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	c.JSON(200, p)
+}

api/publish.go

@@ -0,0 +1,354 @@
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
+	"strings"
+)
+
+// SigningOptions is a shared between publish API GPG options structure
+type SigningOptions struct {
+	Skip           bool
+	Batch          bool
+	GpgKey         string
+	Keyring        string
+	SecretKeyring  string
+	Passphrase     string
+	PassphraseFile string
+}
+
+func getSigner(options *SigningOptions) (utils.Signer, error) {
+	if options.Skip {
+		return nil, nil
+	}
+
+	signer := &utils.GpgSigner{}
+	signer.SetKey(options.GpgKey)
+	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
+	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
+	signer.SetBatch(options.Batch)
+
+	err := signer.Init()
+	if err != nil {
+		return nil, err
+	}
+
+	return signer, nil
+}
+
+// Replace '_' with '/' and double '__' with single '_'
+func parseEscapedPath(path string) string {
+	result := strings.Replace(strings.Replace(path, "_", "/", -1), "//", "_", -1)
+	if result == "" {
+		result = "."
+	}
+	return result
+}
+
+// GET /publish
+func apiPublishList(c *gin.Context) {
+	localCollection := context.CollectionFactory().LocalRepoCollection()
+	localCollection.RLock()
+	defer localCollection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.RLock()
+	defer snapshotCollection.RUnlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	result := make([]*deb.PublishedRepo, 0, collection.Len())
+
+	err := collection.ForEach(func(repo *deb.PublishedRepo) error {
+		err := collection.LoadComplete(repo, context.CollectionFactory())
+		if err != nil {
+			return err
+		}
+
+		result = append(result, repo)
+
+		return nil
+	})
+
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, result)
+}
+
+// POST /publish/:prefix
+func apiPublishRepoOrSnapshot(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+
+	var b struct {
+		SourceKind string `binding:"required"`
+		Sources    []struct {
+			Component string
+			Name      string `binding:"required"`
+		} `binding:"required"`
+		Distribution   string
+		Label          string
+		Origin         string
+		ForceOverwrite bool
+		SkipContents   *bool
+		Architectures  []string
+		Signing        SigningOptions
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	if len(b.Sources) == 0 {
+		c.Fail(400, fmt.Errorf("unable to publish: soures are empty"))
+		return
+	}
+
+	var components []string
+	var sources []interface{}
+
+	if b.SourceKind == "snapshot" {
+		var snapshot *deb.Snapshot
+
+		snapshotCollection := context.CollectionFactory().SnapshotCollection()
+		snapshotCollection.RLock()
+		defer snapshotCollection.RUnlock()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+
+			snapshot, err = snapshotCollection.ByName(source.Name)
+			if err != nil {
+				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			sources = append(sources, snapshot)
+		}
+	} else if b.SourceKind == "local" {
+		var localRepo *deb.LocalRepo
+
+		localCollection := context.CollectionFactory().LocalRepoCollection()
+		localCollection.RLock()
+		defer localCollection.RUnlock()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+
+			localRepo, err = localCollection.ByName(source.Name)
+			if err != nil {
+				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			err = localCollection.LoadComplete(localRepo)
+			if err != nil {
+				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+			}
+
+			sources = append(sources, localRepo)
+		}
+	} else {
+		c.Fail(400, fmt.Errorf("unknown SourceKind"))
+		return
+	}
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, context.CollectionFactory())
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		return
+	}
+	published.Origin = b.Origin
+	published.Label = b.Label
+
+	published.SkipContents = context.Config().SkipContentsPublishing
+	if b.SkipContents != nil {
+		published.SkipContents = *b.SkipContents
+	}
+
+	duplicate := collection.CheckDuplicate(published)
+	if duplicate != nil {
+		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
+		c.Fail(400, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate))
+		return
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		return
+	}
+
+	err = collection.Add(published)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		return
+	}
+
+	c.JSON(201, published)
+}
+
+// PUT /publish/:prefix/:distribution
+func apiPublishUpdateSwitch(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	var b struct {
+		ForceOverwrite bool
+		Signing        SigningOptions
+		SkipContents   *bool
+		Snapshots      []struct {
+			Component string `binding:"required"`
+			Name      string `binding:"required"`
+		}
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	// published.LoadComplete would touch local repo collection
+	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
+	localRepoCollection.RLock()
+	defer localRepoCollection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.RLock()
+	defer snapshotCollection.RUnlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		c.Fail(404, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+	err = collection.LoadComplete(published, context.CollectionFactory())
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	var updatedComponents []string
+
+	if published.SourceKind == "local" {
+		if len(b.Snapshots) > 0 {
+			c.Fail(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
+			return
+		}
+		updatedComponents = published.Components()
+		for _, component := range updatedComponents {
+			published.UpdateLocalRepo(component)
+		}
+	} else if published.SourceKind == "snapshot" {
+		publishedComponents := published.Components()
+		for _, snapshotInfo := range b.Snapshots {
+			if !utils.StrSliceHasItem(publishedComponents, snapshotInfo.Component) {
+				c.Fail(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
+				return
+			}
+
+			snapshot, err := snapshotCollection.ByName(snapshotInfo.Name)
+			if err != nil {
+				c.Fail(404, err)
+				return
+			}
+
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+
+			published.UpdateSnapshot(snapshotInfo.Component, snapshot)
+			updatedComponents = append(updatedComponents, snapshotInfo.Component)
+		}
+	} else {
+		c.Fail(500, fmt.Errorf("unknown published repository type"))
+		return
+	}
+
+	if b.SkipContents != nil {
+		published.SkipContents = *b.SkipContents
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	err = collection.Update(published)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		return
+	}
+
+	err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
+		context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	c.JSON(200, published)
+}
+
+// DELETE /publish/:prefix/:distribution
+func apiPublishDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	// published.LoadComplete would touch local repo collection
+	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
+	localRepoCollection.RLock()
+	defer localRepoCollection.RUnlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	err := collection.Remove(context, storage, prefix, distribution,
+		context.CollectionFactory(), context.Progress(), force)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to drop: %s", err))
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}

api/repos.go

@@ -0,0 +1,365 @@
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
+	"os"
+	"path/filepath"
+)
+
+// GET /api/repos
+func apiReposList(c *gin.Context) {
+	result := []*deb.LocalRepo{}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	context.CollectionFactory().LocalRepoCollection().ForEach(func(r *deb.LocalRepo) error {
+		result = append(result, r)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/repos
+func apiReposCreate(c *gin.Context) {
+	var b struct {
+		Name                string `binding:"required"`
+		Comment             string
+		DefaultDistribution string
+		DefaultComponent    string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	repo := deb.NewLocalRepo(b.Name, b.Comment)
+	repo.DefaultComponent = b.DefaultComponent
+	repo.DefaultDistribution = b.DefaultDistribution
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	err := context.CollectionFactory().LocalRepoCollection().Add(repo)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, repo)
+}
+
+// PUT /api/repos/:name
+func apiReposEdit(c *gin.Context) {
+	var b struct {
+		Comment             string
+		DefaultDistribution string
+		DefaultComponent    string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	if b.Comment != "" {
+		repo.Comment = b.Comment
+	}
+	if b.DefaultDistribution != "" {
+		repo.DefaultDistribution = b.DefaultDistribution
+	}
+	if b.DefaultComponent != "" {
+		repo.DefaultComponent = b.DefaultComponent
+	}
+
+	err = collection.Update(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
+// GET /api/repos/:name
+func apiReposShow(c *gin.Context) {
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
+// DELETE /api/repos/:name
+func apiReposDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.RLock()
+	defer snapshotCollection.RUnlock()
+
+	publishedCollection := context.CollectionFactory().PublishedRepoCollection()
+	publishedCollection.RLock()
+	defer publishedCollection.RUnlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	published := publishedCollection.ByLocalRepo(repo)
+	if len(published) > 0 {
+		c.Fail(409, fmt.Errorf("unable to drop, local repo is published"))
+		return
+	}
+
+	if !force {
+		snapshots := snapshotCollection.ByLocalRepoSource(repo)
+		if len(snapshots) > 0 {
+			c.Fail(409, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override"))
+			return
+		}
+	}
+
+	err = collection.Drop(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// GET /api/repos/:name/packages
+func apiReposPackagesShow(c *gin.Context) {
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	showPackages(c, repo.RefList())
+}
+
+// Handler for both add and delete
+func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p *deb.Package) error) {
+	var b struct {
+		PackageRefs []string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	// verify package refs and build package list
+	for _, ref := range b.PackageRefs {
+		var p *deb.Package
+
+		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
+		if err != nil {
+			if err == database.ErrNotFound {
+				c.Fail(404, fmt.Errorf("package %s: %s", ref, err))
+			} else {
+				c.Fail(500, err)
+			}
+			return
+		}
+		err = cb(list, p)
+		if err != nil {
+			c.Fail(400, err)
+			return
+		}
+	}
+
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save: %s", err))
+		return
+	}
+
+	c.JSON(200, repo)
+
+}
+
+// POST /repos/:name/packages
+func apiReposPackagesAdd(c *gin.Context) {
+	apiReposPackagesAddDelete(c, func(list *deb.PackageList, p *deb.Package) error {
+		return list.Add(p)
+	})
+}
+
+// DELETE /repos/:name/packages
+func apiReposPackagesDelete(c *gin.Context) {
+	apiReposPackagesAddDelete(c, func(list *deb.PackageList, p *deb.Package) error {
+		list.Remove(p)
+		return nil
+	})
+}
+
+// POST /repos/:name/file/:dir/:file
+func apiReposPackageFromFile(c *gin.Context) {
+	// redirect all work to dir method
+	apiReposPackageFromDir(c)
+}
+
+// POST /repos/:name/file/:dir
+func apiReposPackageFromDir(c *gin.Context) {
+	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
+	noRemove := c.Request.URL.Query().Get("noRemove") == "1"
+
+	if !verifyDir(c) {
+		return
+	}
+
+	fileParam := c.Params.ByName("file")
+	if fileParam != "" && !verifyPath(fileParam) {
+		c.Fail(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	verifier := &utils.GpgVerifier{}
+
+	var (
+		sources                      []string
+		packageFiles, failedFiles    []string
+		processedFiles, failedFiles2 []string
+		reporter                     = &aptly.RecordingResultReporter{
+			Warnings:     []string{},
+			AddedLines:   []string{},
+			RemovedLines: []string{},
+		}
+		list *deb.PackageList
+	)
+
+	if fileParam == "" {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"))}
+	} else {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
+	}
+
+	packageFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
+
+	list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to load packages: %s", err))
+		return
+	}
+
+	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+		context.CollectionFactory().PackageCollection(), reporter, nil)
+	failedFiles = append(failedFiles, failedFiles2...)
+
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to import package files: %s", err))
+		return
+	}
+
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save: %s", err))
+		return
+	}
+
+	if !noRemove {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err := os.Remove(file)
+			if err != nil {
+				reporter.Warning("unable to remove file %s: %s", file, err)
+			}
+		}
+
+		// atempt to remove dir, if it fails, that's fine: probably it's not empty
+		os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	}
+
+	if failedFiles == nil {
+		failedFiles = []string{}
+	}
+
+	c.JSON(200, gin.H{
+		"Report":      reporter,
+		"FailedFiles": failedFiles,
+	})
+}

api/router.go

@@ -0,0 +1,112 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+	ctx "github.com/smira/aptly/context"
+	"net/http"
+)
+
+var context *ctx.AptlyContext
+
+// Router returns prebuilt with routes http.Handler
+func Router(c *ctx.AptlyContext) http.Handler {
+	context = c
+
+	router := gin.Default()
+	router.Use(gin.ErrorLogger())
+
+	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
+		// We use a goroutine to count the number of
+		// concurrent requests. When no more requests are
+		// running, we close the database to free the lock.
+		requests := make(chan int)
+		acks := make(chan error)
+
+		go acquireDatabase(requests, acks)
+		go cacheFlusher(requests, acks)
+
+		router.Use(func(c *gin.Context) {
+			requests <- ACQUIREDB
+			err := <-acks
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+			defer func() {
+				requests <- RELEASEDB
+				err = <-acks
+				if err != nil {
+					c.Fail(500, err)
+					return
+				}
+			}()
+			c.Next()
+		})
+
+	} else {
+		go cacheFlusher(nil, nil)
+	}
+
+	root := router.Group("/api")
+
+	{
+		root.GET("/version", apiVersion)
+	}
+
+	{
+		root.GET("/repos", apiReposList)
+		root.POST("/repos", apiReposCreate)
+		root.GET("/repos/:name", apiReposShow)
+		root.PUT("/repos/:name", apiReposEdit)
+		root.DELETE("/repos/:name", apiReposDrop)
+
+		root.GET("/repos/:name/packages", apiReposPackagesShow)
+		root.POST("/repos/:name/packages", apiReposPackagesAdd)
+		root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
+
+		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
+		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+
+		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
+	}
+
+	{
+		root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
+	}
+
+	{
+		root.GET("/files", apiFilesListDirs)
+		root.POST("/files/:dir", apiFilesUpload)
+		root.GET("/files/:dir", apiFilesListFiles)
+		root.DELETE("/files/:dir", apiFilesDeleteDir)
+		root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
+	}
+
+	{
+		root.GET("/publish", apiPublishList)
+		root.POST("/publish", apiPublishRepoOrSnapshot)
+		root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
+		root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
+		root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+	}
+
+	{
+		root.GET("/snapshots", apiSnapshotsList)
+		root.POST("/snapshots", apiSnapshotsCreate)
+		root.PUT("/snapshots/:name", apiSnapshotsUpdate)
+		root.GET("/snapshots/:name", apiSnapshotsShow)
+		root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
+		root.DELETE("/snapshots/:name", apiSnapshotsDrop)
+		root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+	}
+
+	{
+		root.GET("/packages/:key", apiPackagesShow)
+	}
+
+	{
+		root.GET("/graph.:ext", apiGraph)
+	}
+
+	return router
+}

api/snapshot.go

@@ -0,0 +1,410 @@
+package api
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/deb"
+)
+
+// GET /api/snapshots
+func apiSnapshotsList(c *gin.Context) {
+	SortMethodString := c.Request.URL.Query().Get("sort")
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	if SortMethodString == "" {
+		SortMethodString = "name"
+	}
+
+	result := []*deb.Snapshot{}
+	collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
+		result = append(result, snapshot)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/mirrors/:name/snapshots/
+func apiSnapshotsCreateFromMirror(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.RemoteRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().RemoteRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	repo, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		c.Fail(409, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// POST /api/snapshots
+func apiSnapshotsCreate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name            string `binding:"required"`
+		Description     string
+		SourceSnapshots []string
+		PackageRefs     []string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	if b.Description == "" {
+		if len(b.SourceSnapshots)+len(b.PackageRefs) == 0 {
+			b.Description = "Created as empty"
+		}
+	}
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	sources := make([]*deb.Snapshot, len(b.SourceSnapshots))
+
+	for i := range b.SourceSnapshots {
+		sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
+		if err != nil {
+			c.Fail(404, err)
+			return
+		}
+
+		err = snapshotCollection.LoadComplete(sources[i])
+		if err != nil {
+			c.Fail(500, err)
+			return
+		}
+	}
+
+	list := deb.NewPackageList()
+
+	// verify package refs and build package list
+	for _, ref := range b.PackageRefs {
+		var p *deb.Package
+
+		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
+		if err != nil {
+			if err == database.ErrNotFound {
+				c.Fail(404, fmt.Errorf("package %s: %s", ref, err))
+			} else {
+				c.Fail(500, err)
+			}
+			return
+		}
+		err = list.Add(p)
+		if err != nil {
+			c.Fail(400, err)
+			return
+		}
+	}
+
+	snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// POST /api/repos/:name/snapshots
+func apiSnapshotsCreateFromRepository(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.LocalRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	repo, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// PUT /api/snapshots/:name
+func apiSnapshotsUpdate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string
+		Description string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshot, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	_, err = collection.ByName(b.Name)
+	if err == nil {
+		c.Fail(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
+		return
+	}
+
+	if b.Name != "" {
+		snapshot.Name = b.Name
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, snapshot)
+}
+
+// GET /api/snapshots/:name
+func apiSnapshotsShow(c *gin.Context) {
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, snapshot)
+}
+
+// DELETE /api/snapshots/:name
+func apiSnapshotsDrop(c *gin.Context) {
+	name := c.Params.ByName("name")
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	publishedCollection := context.CollectionFactory().PublishedRepoCollection()
+	publishedCollection.RLock()
+	defer publishedCollection.RUnlock()
+
+	snapshot, err := snapshotCollection.ByName(name)
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	published := publishedCollection.BySnapshot(snapshot)
+
+	if len(published) > 0 {
+		c.Fail(409, fmt.Errorf("unable to drop: snapshot is published"))
+		return
+	}
+
+	if !force {
+		snapshots := snapshotCollection.BySnapshotSource(snapshot)
+		if len(snapshots) > 0 {
+			c.Fail(409, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override"))
+			return
+		}
+	}
+
+	err = snapshotCollection.Drop(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// GET /api/snapshots/:name/diff/:withSnapshot
+func apiSnapshotsDiff(c *gin.Context) {
+	onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1"
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshotA, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotA)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotB)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	// Calculate diff
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	result := []deb.PackageDiff{}
+
+	for _, pdiff := range diff {
+		if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) {
+			continue
+		}
+
+		result = append(result, pdiff)
+	}
+
+	c.JSON(200, result)
+}
+
+// GET /api/snapshots/:name/packages
+func apiSnapshotsSearchPackages(c *gin.Context) {
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	showPackages(c, snapshot.RefList())
+}

aptly/report.go

@@ -0,0 +1,67 @@
+package aptly
+
+import (
+	"fmt"
+)
+
+// ResultReporter is abstraction for result reporting from complex processing functions
+type ResultReporter interface {
+	// Warning is non-fatal error message
+	Warning(msg string, a ...interface{})
+	// Removed is signal that something has been removed
+	Removed(msg string, a ...interface{})
+	// Added is signal that something has been added
+	Added(msg string, a ...interface{})
+}
+
+// ConsoleResultReporter is implementation of ResultReporter that prints in colors to console
+type ConsoleResultReporter struct {
+	Progress Progress
+}
+
+// Check interface
+var (
+	_ ResultReporter = &ConsoleResultReporter{}
+)
+
+// Warning is non-fatal error message (yellow)
+func (c *ConsoleResultReporter) Warning(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@y[!]@| @!"+msg+"@|", a...)
+}
+
+// Removed is signal that something has been removed (red)
+func (c *ConsoleResultReporter) Removed(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@r[-]@| "+msg, a...)
+}
+
+// Added is signal that something has been added (green)
+func (c *ConsoleResultReporter) Added(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@g[+]@| "+msg, a...)
+}
+
+// RecordingResultReporter is implementation of ResultReporter that collects all messages
+type RecordingResultReporter struct {
+	Warnings     []string
+	AddedLines   []string `json:"Added"`
+	RemovedLines []string `json:"Removed"`
+}
+
+// Check interface
+var (
+	_ ResultReporter = &RecordingResultReporter{}
+)
+
+// Warning is non-fatal error message
+func (r *RecordingResultReporter) Warning(msg string, a ...interface{}) {
+	r.Warnings = append(r.Warnings, fmt.Sprintf(msg, a...))
+}
+
+// Removed is signal that something has been removed
+func (r *RecordingResultReporter) Removed(msg string, a ...interface{}) {
+	r.RemovedLines = append(r.RemovedLines, fmt.Sprintf(msg, a...))
+}
+
+// Added is signal that something has been added
+func (r *RecordingResultReporter) Added(msg string, a ...interface{}) {
+	r.AddedLines = append(r.AddedLines, fmt.Sprintf(msg, a...))
+}

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
 // Version of aptly
-const Version = "0.8"
+const Version = "0.9.7"
 
 // Enable debugging features?
 const EnableDebug = false

cmd/api.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdAPI() *commander.Command {
+	return &commander.Command{
+		UsageLine: "api",
+		Short:     "start API server/issue requests",
+		Subcommands: []*commander.Command{
+			makeCmdAPIServe(),
+		},
+	}
+}

cmd/api_serve.go

@@ -0,0 +1,53 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/api"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"net/http"
+)
+
+func aptlyAPIServe(cmd *commander.Command, args []string) error {
+	var (
+		err error
+	)
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	listen := context.Flags().Lookup("listen").Value.String()
+
+	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
+
+	err = http.ListenAndServe(listen, api.Router(context))
+	if err != nil {
+		return fmt.Errorf("unable to serve: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdAPIServe() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyAPIServe,
+		UsageLine: "serve",
+		Short:     "start API HTTP service",
+		Long: `
+Stat HTTP server with aptly REST API.
+
+Example:
+
+  $ aptly api serve -listen=:8080
+`,
+		Flag: *flag.NewFlagSet("aptly-serve", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening")
+	cmd.Flag.Bool("no-lock", false, "don't lock the database")
+
+	return cmd
+
+}

cmd/cmd.go

@@ -2,12 +2,14 @@
 package cmd
 
 import (
+	"bytes"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"os"
+	"text/template"
 	"time"
 )
 
@@ -34,6 +36,32 @@ func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 	return
 }
 
+// PrintPackageList shows package list with specified format or default representation
+func PrintPackageList(result *deb.PackageList, format string) error {
+	if format == "" {
+		return result.ForEach(func(p *deb.Package) error {
+			context.Progress().Printf("%s\n", p)
+			return nil
+		})
+	}
+
+	formatTemplate, err := template.New("format").Parse(format)
+	if err != nil {
+		return fmt.Errorf("error parsing -format template: %s", err)
+	}
+
+	return result.ForEach(func(p *deb.Package) error {
+		b := &bytes.Buffer{}
+		err = formatTemplate.Execute(b, p.ExtendedStanza())
+		if err != nil {
+			return fmt.Errorf("error applying template: %s", err)
+		}
+		context.Progress().Printf("%s\n", b.String())
+		return nil
+	})
+
+}
+
 // LookupOption checks boolean flag with default (usually config) and command-line
 // setting
 func LookupOption(defaultValue bool, flags *flag.FlagSet, name string) (result bool) {
@@ -65,24 +93,25 @@ fine-grained changes in repository contents to transition your
 package environment to new version.`,
 		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
 		Subcommands: []*commander.Command{
+			makeCmdConfig(),
 			makeCmdDb(),
 			makeCmdGraph(),
 			makeCmdMirror(),
 			makeCmdRepo(),
 			makeCmdServe(),
 			makeCmdSnapshot(),
-			// Disabled on no docs
+			makeCmdTask(),
-			//makeCmdTask(),
 			makeCmdPublish(),
 			makeCmdVersion(),
 			makeCmdPackage(),
+			makeCmdAPI(),
 		},
 	}
 
 	cmd.Flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests")
 	cmd.Flag.Bool("dep-follow-source", false, "when processing dependencies, follow from binary to Source packages")
 	cmd.Flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends")
-	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if depdency is 'a|b'")
+	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
 	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
 

cmd/config.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdConfig() *commander.Command {
+	return &commander.Command{
+		UsageLine: "config",
+		Short:     "manage aptly configuration",
+		Subcommands: []*commander.Command{
+			makeCmdConfigShow(),
+		},
+	}
+}

cmd/config_show.go

@@ -0,0 +1,38 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/smira/commander"
+)
+
+func aptlyConfigShow(cmd *commander.Command, args []string) error {
+
+	config := context.Config()
+	prettyJSON, err := json.MarshalIndent(config, "", "    ")
+
+	if err != nil {
+		return fmt.Errorf("unable to dump the config file: %s", err)
+	}
+
+	fmt.Println(string(prettyJSON))
+
+	return nil
+}
+
+func makeCmdConfigShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyConfigShow,
+		UsageLine: "show",
+		Short:     "show current aptly's config",
+		Long: `
+Command show displays the current aptly configuration.
+
+Example:
+
+  $ aptly config show
+
+`,
+	}
+	return cmd
+}

cmd/context.go

@@ -1,313 +1,20 @@
 package cmd
 
 import (
-	"fmt"
+	ctx "github.com/smira/aptly/context"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/console"
-	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/files"
-	"github.com/smira/aptly/http"
-	"github.com/smira/aptly/s3"
-	"github.com/smira/aptly/utils"
-	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"path/filepath"
-	"runtime"
-	"runtime/pprof"
-	"strings"
-	"time"
 )
 
-// AptlyContext is a common context shared by all commands
+var context *ctx.AptlyContext
-type AptlyContext struct {
-	flags, globalFlags *flag.FlagSet
-	configLoaded       bool
-
-	progress          aptly.Progress
-	downloader        aptly.Downloader
-	database          database.Storage
-	packagePool       aptly.PackagePool
-	publishedStorages map[string]aptly.PublishedStorage
-	collectionFactory *deb.CollectionFactory
-	dependencyOptions int
-	architecturesList []string
-	// Debug features
-	fileCPUProfile *os.File
-	fileMemProfile *os.File
-	fileMemStats   *os.File
-}
-
-var context *AptlyContext
-
-// Check interface
-var _ aptly.PublishedStorageProvider = &AptlyContext{}
-
-// FatalError is type for panicking to abort execution with non-zero
-// exit code and print meaningful explanation
-type FatalError struct {
-	ReturnCode int
-	Message    string
-}
-
-// Fatal panics and aborts execution with exit code 1
-func Fatal(err error) {
-	returnCode := 1
-	if err == commander.ErrFlagError || err == commander.ErrCommandError {
-		returnCode = 2
-	}
-	panic(&FatalError{ReturnCode: returnCode, Message: err.Error()})
-}
-
-// Config loads and returns current configuration
-func (context *AptlyContext) Config() *utils.ConfigStructure {
-	if !context.configLoaded {
-		var err error
-
-		configLocation := context.globalFlags.Lookup("config").Value.String()
-		if configLocation != "" {
-			err = utils.LoadConfig(configLocation, &utils.Config)
-
-			if err != nil {
-				Fatal(err)
-			}
-		} else {
-			configLocations := []string{
-				filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
-				"/etc/aptly.conf",
-			}
-
-			for _, configLocation := range configLocations {
-				err = utils.LoadConfig(configLocation, &utils.Config)
-				if err == nil {
-					break
-				}
-				if !os.IsNotExist(err) {
-					Fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
-				}
-			}
-
-			if err != nil {
-				fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
-				utils.SaveConfig(configLocations[0], &utils.Config)
-			}
-		}
-
-		context.configLoaded = true
-
-	}
-	return &utils.Config
-}
-
-// DependencyOptions calculates options related to dependecy handling
-func (context *AptlyContext) DependencyOptions() int {
-	if context.dependencyOptions == -1 {
-		context.dependencyOptions = 0
-		if LookupOption(context.Config().DepFollowSuggests, context.globalFlags, "dep-follow-suggests") {
-			context.dependencyOptions |= deb.DepFollowSuggests
-		}
-		if LookupOption(context.Config().DepFollowRecommends, context.globalFlags, "dep-follow-recommends") {
-			context.dependencyOptions |= deb.DepFollowRecommends
-		}
-		if LookupOption(context.Config().DepFollowAllVariants, context.globalFlags, "dep-follow-all-variants") {
-			context.dependencyOptions |= deb.DepFollowAllVariants
-		}
-		if LookupOption(context.Config().DepFollowSource, context.globalFlags, "dep-follow-source") {
-			context.dependencyOptions |= deb.DepFollowSource
-		}
-	}
-
-	return context.dependencyOptions
-}
-
-// ArchitecturesList returns list of architectures fixed via command line or config
-func (context *AptlyContext) ArchitecturesList() []string {
-	if context.architecturesList == nil {
-		context.architecturesList = context.Config().Architectures
-		optionArchitectures := context.globalFlags.Lookup("architectures").Value.String()
-		if optionArchitectures != "" {
-			context.architecturesList = strings.Split(optionArchitectures, ",")
-		}
-	}
-
-	return context.architecturesList
-}
-
-// Progress creates or returns Progress object
-func (context *AptlyContext) Progress() aptly.Progress {
-	if context.progress == nil {
-		context.progress = console.NewProgress()
-		context.progress.Start()
-	}
-
-	return context.progress
-}
-
-// Downloader returns instance of current downloader
-func (context *AptlyContext) Downloader() aptly.Downloader {
-	if context.downloader == nil {
-		var downloadLimit int64
-		limitFlag := context.flags.Lookup("download-limit")
-		if limitFlag != nil {
-			downloadLimit = limitFlag.Value.Get().(int64)
-		}
-		if downloadLimit == 0 {
-			downloadLimit = context.Config().DownloadLimit
-		}
-		context.downloader = http.NewDownloader(context.Config().DownloadConcurrency,
-			downloadLimit*1024, context.Progress())
-	}
-
-	return context.downloader
-}
-
-// DBPath builds path to database
-func (context *AptlyContext) DBPath() string {
-	return filepath.Join(context.Config().RootDir, "db")
-}
-
-// Database opens and returns current instance of database
-func (context *AptlyContext) Database() (database.Storage, error) {
-	if context.database == nil {
-		var err error
-
-		context.database, err = database.OpenDB(context.DBPath())
-		if err != nil {
-			return nil, fmt.Errorf("can't open database: %s", err)
-		}
-	}
-
-	return context.database, nil
-}
-
-// CloseDatabase closes the db temporarily
-func (context *AptlyContext) CloseDatabase() error {
-	if context.database == nil {
-		return nil
-	}
-
-	return context.database.Close()
-}
-
-// ReOpenDatabase reopens the db after close
-func (context *AptlyContext) ReOpenDatabase() error {
-	if context.database == nil {
-		return nil
-	}
-
-	const MaxTries = 10
-	const Delay = 10 * time.Second
-
-	for try := 0; try < MaxTries; try++ {
-		err := context.database.ReOpen()
-		if err == nil || strings.Index(err.Error(), "resource temporarily unavailable") == -1 {
-			return err
-		}
-		context.Progress().Printf("Unable to reopen database, sleeping %s\n", Delay)
-		<-time.After(Delay)
-	}
-
-	return fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
-}
-
-// CollectionFactory builds factory producing all kinds of collections
-func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
-	if context.collectionFactory == nil {
-		db, err := context.Database()
-		if err != nil {
-			Fatal(err)
-		}
-		context.collectionFactory = deb.NewCollectionFactory(db)
-	}
-
-	return context.collectionFactory
-}
-
-// PackagePool returns instance of PackagePool
-func (context *AptlyContext) PackagePool() aptly.PackagePool {
-	if context.packagePool == nil {
-		context.packagePool = files.NewPackagePool(context.Config().RootDir)
-	}
-
-	return context.packagePool
-}
-
-// GetPublishedStorage returns instance of PublishedStorage
-func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
-	publishedStorage, ok := context.publishedStorages[name]
-	if !ok {
-		if name == "" {
-			publishedStorage = files.NewPublishedStorage(context.Config().RootDir)
-		} else if strings.HasPrefix(name, "s3:") {
-			params, ok := context.Config().S3PublishRoots[name[3:]]
-			if !ok {
-				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
-			}
-
-			var err error
-			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
-				params.Region, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
-				params.EncryptionMethod, params.PlusWorkaround)
-			if err != nil {
-				Fatal(err)
-			}
-		} else {
-			Fatal(fmt.Errorf("unknown published storage format: %v", name))
-		}
-		context.publishedStorages[name] = publishedStorage
-	}
-
-	return publishedStorage
-}
-
-// UpdateFlags sets internal copy of flags in the context
-func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) {
-	context.flags = flags
-}
 
 // ShutdownContext shuts context down
 func ShutdownContext() {
-	if aptly.EnableDebug {
+	context.Shutdown()
-		if context.fileMemProfile != nil {
-			pprof.WriteHeapProfile(context.fileMemProfile)
-			context.fileMemProfile.Close()
-			context.fileMemProfile = nil
-		}
-		if context.fileCPUProfile != nil {
-			pprof.StopCPUProfile()
-			context.fileCPUProfile.Close()
-			context.fileCPUProfile = nil
-		}
-		if context.fileMemProfile != nil {
-			context.fileMemProfile.Close()
-			context.fileMemProfile = nil
-		}
-	}
-	if context.database != nil {
-		context.database.Close()
-		context.database = nil
-	}
-	if context.downloader != nil {
-		context.downloader.Abort()
-		context.downloader = nil
-	}
-	if context.progress != nil {
-		context.progress.Shutdown()
-		context.progress = nil
-	}
 }
 
 // CleanupContext does partial shutdown of context
 func CleanupContext() {
-	if context.downloader != nil {
+	context.Cleanup()
-		context.downloader.Shutdown()
-		context.downloader = nil
-	}
-	if context.progress != nil {
-		context.progress.Shutdown()
-		context.progress = nil
-	}
 }
 
 // InitContext initializes context with default settings
@@ -318,60 +25,7 @@ func InitContext(flags *flag.FlagSet) error {
 		panic("context already initialized")
 	}
 
-	context = &AptlyContext{
+	context, err = ctx.NewContext(flags)
-		flags:             flags,
-		globalFlags:       flags,
-		dependencyOptions: -1,
-		publishedStorages: map[string]aptly.PublishedStorage{},
-	}
 
-	if aptly.EnableDebug {
+	return err
-		cpuprofile := flags.Lookup("cpuprofile").Value.String()
-		if cpuprofile != "" {
-			context.fileCPUProfile, err = os.Create(cpuprofile)
-			if err != nil {
-				return err
-			}
-			pprof.StartCPUProfile(context.fileCPUProfile)
-		}
-
-		memprofile := flags.Lookup("memprofile").Value.String()
-		if memprofile != "" {
-			context.fileMemProfile, err = os.Create(memprofile)
-			if err != nil {
-				return err
-			}
-		}
-
-		memstats := flags.Lookup("memstats").Value.String()
-		if memstats != "" {
-			interval := flags.Lookup("meminterval").Value.Get().(time.Duration)
-
-			context.fileMemStats, err = os.Create(memstats)
-			if err != nil {
-				return err
-			}
-
-			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
-
-			go func() {
-				var stats runtime.MemStats
-
-				start := time.Now().UnixNano()
-
-				for {
-					runtime.ReadMemStats(&stats)
-					if context.fileMemStats != nil {
-						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
-							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
-						time.Sleep(interval)
-					} else {
-						break
-					}
-				}
-			}()
-		}
-	}
-
-	return nil
 }

cmd/db_cleanup.go

@@ -6,6 +6,7 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"sort"
+	"strings"
 )
 
 // aptly db cleanup
@@ -17,31 +18,98 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
+	verbose := context.Flags().Lookup("verbose").Value.Get().(bool)
+	dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
+
 	// collect information about references packages...
 	existingPackageRefs := deb.NewPackageRefList()
 
-	context.Progress().Printf("Loading mirrors, local repos and snapshots...\n")
+	// used only in verbose mode to report package use source
+	packageRefSources := map[string][]string{}
+
+	context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|")
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
+	}
 	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
 		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
 		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("mirror %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
+
 		return nil
 	})
 	if err != nil {
 		return err
 	}
 
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
+	}
 	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
 		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return err
 		}
+
 		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("local repo %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
+	}
+	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
+		}
+
+		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+
+		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+
+		if verbose {
+			description := fmt.Sprintf("snapshot %s", snapshot.Name)
+			snapshot.RefList().ForEach(func(key []byte) error {
+				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+				return nil
+			})
 		}
 		return nil
 	})
@@ -49,12 +117,32 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+	if verbose {
-		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
+	}
+	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
+		}
+		if published.SourceKind != "local" {
+			return nil
+		}
+		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
 		if err != nil {
 			return err
 		}
-		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false)
+
+		for _, component := range published.Components() {
+			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+			if verbose {
+				description := fmt.Sprintf("published repository %s:%s/%s component %s",
+					published.Storage, published.Prefix, published.Distribution, component)
+				published.RefList(component).ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
+		}
 		return nil
 	})
 	if err != nil {
@@ -62,38 +150,65 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	}
 
 	// ... and compare it to the list of all packages
-	context.Progress().Printf("Loading list of all packages...\n")
+	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
 	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
 
 	toDelete := allPackageRefs.Substract(existingPackageRefs)
 
 	// delete packages that are no longer referenced
-	context.Progress().Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())
 
 	// database can't err as collection factory already constructed
 	db, _ := context.Database()
-	db.StartBatch()
-	err = toDelete.ForEach(func(ref []byte) error {
-		return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
-	})
-	if err != nil {
-		return err
-	}
 
-	err = db.FinishBatch()
+	if toDelete.Len() > 0 {
-	if err != nil {
+		if verbose {
-		return fmt.Errorf("unable to write to DB: %s", err)
+			context.Progress().ColoredPrintf("@{r}List of package keys to delete:@|")
+			err = toDelete.ForEach(func(ref []byte) error {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", string(ref))
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if !dryRun {
+			db.StartBatch()
+			err = toDelete.ForEach(func(ref []byte) error {
+				return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
+			})
+			if err != nil {
+				return err
+			}
+
+			err = db.FinishBatch()
+			if err != nil {
+				return fmt.Errorf("unable to write to DB: %s", err)
+			}
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped deletion, as -dry-run has been requested.@|")
+		}
 	}
 
 	// now, build a list of files that should be present in Repository (package pool)
-	context.Progress().Printf("Building list of files referenced by packages...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())
 	context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
 
 	err = existingPackageRefs.ForEach(func(key []byte) error {
 		pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
 		if err2 != nil {
-			return err2
+			tail := ""
+			if verbose {
+				tail = fmt.Sprintf(" (sources: %s)", strings.Join(packageRefSources[string(key)], ", "))
+			}
+			if dryRun {
+				context.Progress().ColoredPrintf("@{r!}Unresolvable package reference, skipping (-dry-run): %s: %s%s",
+					string(key), err2, tail)
+				return nil
+			}
+			return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
 		}
 		paths, err2 := pkg.FilepathList(context.PackagePool())
 		if err2 != nil {
@@ -112,7 +227,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	context.Progress().ShutdownBar()
 
 	// build a list of files in the package pool
-	context.Progress().Printf("Building list of files in package pool...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files in package pool...@|")
 	existingFiles, err := context.PackagePool().FilepathList(context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to collect file paths: %s", err)
@@ -122,28 +237,43 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
 
 	// delete files that are no longer referenced
-	context.Progress().Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced files (%d)...@|", len(filesToDelete))
 
 	if len(filesToDelete) > 0 {
-		context.Progress().InitBar(int64(len(filesToDelete)), false)
+		if verbose {
-
+			context.Progress().ColoredPrintf("@{r}List of files to be deleted:@|")
-		var size, totalSize int64
+			for _, file := range filesToDelete {
-		for _, file := range filesToDelete {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", file)
-			size, err = context.PackagePool().Remove(file)
-			if err != nil {
-				return err
 			}
-
-			context.Progress().AddBar(1)
-			totalSize += size
 		}
-		context.Progress().ShutdownBar()
 
-		context.Progress().Printf("Disk space freed: %s...\n", utils.HumanBytes(totalSize))
+		if !dryRun {
+			context.Progress().InitBar(int64(len(filesToDelete)), false)
+
+			var size, totalSize int64
+			for _, file := range filesToDelete {
+				size, err = context.PackagePool().Remove(file)
+				if err != nil {
+					return err
+				}
+
+				context.Progress().AddBar(1)
+				totalSize += size
+			}
+			context.Progress().ShutdownBar()
+
+			context.Progress().ColoredPrintf("@{w!}Disk space freed: %s...@|", utils.HumanBytes(totalSize))
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped file deletion, as -dry-run has been requested.@|")
+		}
 	}
 
-	context.Progress().Printf("Compacting database...\n")
+	if !dryRun {
-	err = db.CompactDB()
+		context.Progress().ColoredPrintf("@{w!}Compacting database...@|")
+		err = db.CompactDB()
+	} else {
+		context.Progress().ColoredPrintf("@{y!}Skipped DB compaction, as -dry-run has been requested.@|")
+	}
 
 	return err
 }
@@ -163,5 +293,8 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("verbose", false, "be verbose when loading objects/removing them")
+	cmd.Flag.Bool("dry-run", false, "don't delete anything")
+
 	return cmd
 }

cmd/graph.go

@@ -2,15 +2,15 @@ package cmd
 
 import (
 	"bytes"
-	"code.google.com/p/gographviz"
 	"fmt"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"strings"
+	"path/filepath"
 )
 
 func aptlyGraph(cmd *commander.Command, args []string) error {
@@ -21,121 +21,11 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	graph := gographviz.NewEscape()
-	graph.SetDir(true)
-	graph.SetName("aptly")
-
-	existingNodes := map[string]bool{}
-
-	fmt.Printf("Loading mirrors...\n")
-
-	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
-		}
-
-		graph.AddNode("aptly", repo.UUID, map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "darkgoldenrod1",
-			"label": fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
-				repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "),
-				strings.Join(repo.Architectures, ", "), repo.NumPackages()),
-		})
-		existingNodes[repo.UUID] = true
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Loading local repos...\n")
-
-	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
-		}
-
-		graph.AddNode("aptly", repo.UUID, map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "mediumseagreen",
-			"label": fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
-				repo.Name, repo.Comment, repo.NumPackages()),
-		})
-		existingNodes[repo.UUID] = true
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Loading snapshots...\n")
-
-	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		existingNodes[snapshot.UUID] = true
-		return nil
-	})
-
-	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
-			return err
-		}
-
-		description := snapshot.Description
-		if snapshot.SourceKind == "repo" {
-			description = "Snapshot from repo"
-		}
-
-		graph.AddNode("aptly", snapshot.UUID, map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "cadetblue1",
-			"label":     fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages()),
-		})
-
-		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
-			for _, uuid := range snapshot.SourceIDs {
-				_, exists := existingNodes[uuid]
-				if exists {
-					graph.AddEdge(uuid, snapshot.UUID, true, nil)
-				}
-			}
-		}
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Loading published repos...\n")
-
-	context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		graph.AddNode("aptly", repo.UUID, map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "darkolivegreen1",
-			"label": fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution,
-				strings.Join(repo.Components(), " "), strings.Join(repo.Architectures, ", ")),
-		})
-
-		for _, uuid := range repo.Sources {
-			_, exists := existingNodes[uuid]
-			if exists {
-				graph.AddEdge(uuid, repo.UUID, true, nil)
-			}
-		}
-
-		return nil
-	})
-
 	fmt.Printf("Generating graph...\n")
+	graph, err := deb.BuildGraph(context.CollectionFactory())
+	if err != nil {
+		return err
+	}
 
 	buf := bytes.NewBufferString(graph.String())
 
@@ -146,9 +36,16 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 	tempfile.Close()
 	os.Remove(tempfile.Name())
 
-	tempfilename := tempfile.Name() + ".png"
+	format := context.Flags().Lookup("format").Value.String()
+	output := context.Flags().Lookup("output").Value.String()
 
-	command := exec.Command("dot", "-Tpng", "-o"+tempfilename)
+	if filepath.Ext(output) != "" {
+		format = filepath.Ext(output)[1:]
+	}
+
+	tempfilename := tempfile.Name() + "." + format
+
+	command := exec.Command("dot", "-T"+format, "-o"+tempfilename)
 	command.Stderr = os.Stderr
 
 	stdin, err := command.StdinPipe()
@@ -176,10 +73,18 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	err = exec.Command("open", tempfilename).Run()
+	if output != "" {
-	if err != nil {
+		err = utils.CopyFile(tempfilename, output)
-		fmt.Printf("Rendered to PNG file: %s\n", tempfilename)
+		if err != nil {
-		err = nil
+			return fmt.Errorf("unable to copy %s -> %s: %s", tempfilename, output, err)
+		}
+		_ = os.Remove(tempfilename)
+
+		fmt.Printf("Output saved to %s\n", output)
+	} else {
+		fmt.Printf("Rendered to %s file: %s, trying to open it...\n", format, tempfilename)
+
+		_ = exec.Command("open", tempfilename).Run()
 	}
 
 	return err
@@ -201,5 +106,8 @@ Example:
 `,
 	}
 
+	cmd.Flag.String("format", "png", "render graph to specified format (png, svg, pdf, etc.)")
+	cmd.Flag.String("output", "", "specify output filename, default is to open result in viewer")
+
 	return cmd
 }

cmd/mirror_create.go

@@ -16,8 +16,8 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.flags, "with-sources")
+	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
-	downloadUdebs := context.flags.Lookup("with-udebs").Value.Get().(bool)
+	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -40,8 +40,9 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
-	repo.Filter = context.flags.Lookup("filter").Value.String()
+	repo.Filter = context.Flags().Lookup("filter").Value.String()
-	repo.FilterWithDeps = context.flags.Lookup("filter-with-deps").Value.Get().(bool)
+	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
+	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
 
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
@@ -50,7 +51,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	verifier, err := getVerifier(context.flags)
+	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
@@ -95,6 +96,7 @@ Example:
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/mirror_drop.go

@@ -25,7 +25,7 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
 		snapshots := context.CollectionFactory().SnapshotCollection().ByRemoteRepoSource(repo)
 

cmd/mirror_edit.go

@@ -24,7 +24,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	context.flags.Visit(func(flag *flag.Flag) {
+	context.Flags().Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
 			repo.Filter = flag.Value.String()
@@ -48,7 +48,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	if context.globalFlags.Lookup("architectures").Value.String() != "" {
+	if context.GlobalFlags().Lookup("architectures").Value.String() != "" {
 		repo.Architectures = context.ArchitecturesList()
 
 		err = repo.Fetch(context.Downloader(), nil)

cmd/mirror_list.go

@@ -28,6 +28,8 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
+	context.CloseDatabase()
+
 	sort.Strings(repos)
 
 	if raw {

cmd/mirror_search.go

@@ -21,6 +21,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/mirror_show.go

@@ -66,7 +66,7 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		fmt.Printf("%s: %s\n", k, repo.Meta[k])
 	}
 
-	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		if repo.LastDownloadDate.IsZero() {
 			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")

cmd/mirror_update.go

@@ -31,7 +31,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
 		err = repo.CheckLock()
 		if err != nil {
@@ -39,9 +39,9 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	ignoreMismatch := context.flags.Lookup("ignore-checksums").Value.Get().(bool)
+	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
 
-	verifier, err := getVerifier(context.flags)
+	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}

cmd/package_search.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -21,10 +20,12 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 	}
 
 	result := q.Query(context.CollectionFactory().PackageCollection())
-	result.ForEach(func(p *deb.Package) error {
+	if result.Len() == 0 {
-		context.Progress().Printf("%s\n", p)
+		return fmt.Errorf("no results")
-		return nil
+	}
-	})
+
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format)
 
 	return err
 }
@@ -44,5 +45,7 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-package-search", flag.ExitOnError),
 	}
 
+	cmd.Flag.String("format", "", "custom format for result printing")
+
 	return cmd
 }

cmd/package_show.go

@@ -72,15 +72,15 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	withFiles := context.flags.Lookup("with-files").Value.Get().(bool)
+	withFiles := context.Flags().Lookup("with-files").Value.Get().(bool)
-	withReferences := context.flags.Lookup("with-references").Value.Get().(bool)
+	withReferences := context.Flags().Lookup("with-references").Value.Get().(bool)
 
 	w := bufio.NewWriter(os.Stdout)
 
 	result := q.Query(context.CollectionFactory().PackageCollection())
 
 	err = result.ForEach(func(p *deb.Package) error {
-		p.Stanza().WriteTo(w)
+		p.Stanza().WriteTo(w, p.IsSource, false)
 		w.Flush()
 		fmt.Printf("\n")
 
@@ -116,7 +116,7 @@ func makeCmdPackageShow() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPackageShow,
 		UsageLine: "show <package-query>",
-		Short:     "show details about packages matcing query",
+		Short:     "show details about packages matching query",
 		Long: `
 Command shows displays detailed meta-information about packages
 matching query. Information from Debian control file is displayed.

cmd/publish.go

@@ -4,7 +4,6 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
@@ -16,6 +15,7 @@ func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
 	signer.SetKey(flags.Lookup("gpg-key").Value.String())
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
 	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
+	signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
 
 	err := signer.Init()
 	if err != nil {
@@ -26,20 +26,6 @@ func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
 
 }
 
-func parsePrefix(param string) (storage, prefix string) {
-	i := strings.LastIndex(param, ":")
-	if i != -1 {
-		storage = param[:i]
-		prefix = param[i+1:]
-		if prefix == "" {
-			prefix = "."
-		}
-	} else {
-		prefix = param
-	}
-	return
-}
-
 func makeCmdPublish() *commander.Command {
 	return &commander.Command{
 		UsageLine: "publish",

cmd/publish_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -19,10 +20,10 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 		param = args[1]
 	}
 
-	storage, prefix := parsePrefix(param)
+	storage, prefix := deb.ParsePrefix(param)
 
 	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress())
+		context.CollectionFactory(), context.Progress(), context.Flags().Lookup("force-drop").Value.Get().(bool))
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -47,5 +48,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
+
 	return cmd
 }

cmd/publish_list.go

@@ -36,6 +36,8 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load list of repos: %s", err)
 	}
 
+	context.CloseDatabase()
+
 	sort.Strings(published)
 
 	if raw {

cmd/publish_repo.go

@@ -39,7 +39,9 @@ Example:
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")

cmd/publish_snapshot.go

@@ -13,7 +13,7 @@ import (
 func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	var err error
 
-	components := strings.Split(context.flags.Lookup("component").Value.String(), ",")
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
 
 	if len(args) < len(components) || len(args) > len(components)+1 {
 		cmd.Usage()
@@ -27,7 +27,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	} else {
 		param = ""
 	}
-	storage, prefix := parsePrefix(param)
+	storage, prefix := deb.ParsePrefix(param)
 
 	var (
 		sources = []interface{}{}
@@ -110,14 +110,20 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		panic("unknown command")
 	}
 
-	distribution := context.flags.Lookup("distribution").Value.String()
+	distribution := context.Flags().Lookup("distribution").Value.String()
 
 	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
-	published.Origin = cmd.Flag.Lookup("origin").Value.String()
+	published.Origin = context.Flags().Lookup("origin").Value.String()
-	published.Label = cmd.Flag.Lookup("label").Value.String()
+	published.Label = context.Flags().Lookup("label").Value.String()
+
+	published.SkipContents = context.Config().SkipContentsPublishing
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
 
 	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
@@ -125,12 +131,12 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 	}
 
-	signer, err := getSigner(context.flags)
+	signer, err := getSigner(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
 	if forceOverwrite {
 		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
 			"the same package pool.\n")
@@ -201,7 +207,9 @@ Example:
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")

cmd/publish_switch.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"strings"
@@ -11,7 +12,7 @@ import (
 func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	var err error
 
-	components := strings.Split(context.flags.Lookup("component").Value.String(), ",")
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
 
 	if len(args) < len(components)+1 || len(args) > len(components)+2 {
 		cmd.Usage()
@@ -33,7 +34,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		names = args[1:]
 	}
 
-	storage, prefix := parsePrefix(param)
+	storage, prefix := deb.ParsePrefix(param)
 
 	var published *deb.PublishedRepo
 
@@ -61,6 +62,10 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	}
 
 	for i, component := range components {
+		if !utils.StrSliceHasItem(publishedComponents, component) {
+			return fmt.Errorf("unable to switch: component %s is not in published repository", component)
+		}
+
 		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(names[i])
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
@@ -74,17 +79,21 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		published.UpdateSnapshot(component, snapshot)
 	}
 
-	signer, err := getSigner(context.flags)
+	signer, err := getSigner(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
 	if forceOverwrite {
 		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
 			"the same package pool.\n")
 	}
 
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
@@ -112,7 +121,7 @@ func makeCmdPublishSwitch() *commander.Command {
 		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
 		Short:     "update published repository by switching to new snapshot",
 		Long: `
-Command switches in-place published repository with new snapshot contents. All
+Command switches in-place published snapshots with new snapshot contents. All
 publishing parameters are preserved (architecture list, distribution,
 component).
 
@@ -120,11 +129,14 @@ For multiple component repositories, flag -component should be given with
 list of components to update. Corresponding snapshots should be given in the
 same order, e.g.:
 
-	aptly publish update -component=main,contrib wheezy wh-main wh-contrib
+	aptly publish switch -component=main,contrib wheezy wh-main wh-contrib
 
 Example:
 
-    $ aptly publish update wheezy ppa wheezy-7.5
+    $ aptly publish switch wheezy ppa wheezy-7.5
+
+This command would switch published repository (with one component) named ppa/wheezy
+(prefix ppa, dsitribution wheezy to new snapshot wheezy-7.5).
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
 	}
@@ -133,7 +145,9 @@ Example:
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 

cmd/publish_update.go

@@ -20,7 +20,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 	if len(args) == 2 {
 		param = args[1]
 	}
-	storage, prefix := parsePrefix(param)
+	storage, prefix := deb.ParsePrefix(param)
 
 	var published *deb.PublishedRepo
 
@@ -43,17 +43,21 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		published.UpdateLocalRepo(component)
 	}
 
-	signer, err := getSigner(context.flags)
+	signer, err := getSigner(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	forceOverwrite := context.flags.Lookup("force-overwrite").Value.Get().(bool)
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
 	if forceOverwrite {
 		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
 			"the same package pool.\n")
 	}
 
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
@@ -100,7 +104,9 @@ Example:
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd

cmd/repo.go

@@ -21,6 +21,7 @@ func makeCmdRepo() *commander.Command {
 			makeCmdRepoShow(),
 			makeCmdRepoRename(),
 			makeCmdRepoSearch(),
+			makeCmdRepoInclude(),
 		},
 	}
 }

cmd/repo_add.go

@@ -2,14 +2,12 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 	"os"
-	"path/filepath"
-	"sort"
-	"strings"
 )
 
 func aptlyRepoAdd(cmd *commander.Command, args []string) error {
@@ -40,151 +38,19 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	forceReplace := context.flags.Lookup("force-replace").Value.Get().(bool)
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 
-	packageFiles := []string{}
+	var packageFiles, failedFiles []string
-	failedFiles := []string{}
 
-	for _, location := range args[1:] {
+	packageFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
-		info, err2 := os.Stat(location)
-		if err2 != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to process %s: %s@|", location, err2)
-			failedFiles = append(failedFiles, location)
-			continue
-		}
-		if info.IsDir() {
-			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
-				if err3 != nil {
-					return err3
-				}
-				if info.IsDir() {
-					return nil
-				}
 
-				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+	var processedFiles, failedFiles2 []string
-					strings.HasSuffix(info.Name(), ".dsc") {
-					packageFiles = append(packageFiles, path)
-				}
 
-				return nil
+	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-			})
+		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil)
-		} else {
+	failedFiles = append(failedFiles, failedFiles2...)
-			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+	if err != nil {
-				strings.HasSuffix(info.Name(), ".dsc") {
+		return fmt.Errorf("unable to import package files: %s", err)
-				packageFiles = append(packageFiles, location)
-			} else {
-				context.Progress().ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
-				failedFiles = append(failedFiles, location)
-				continue
-			}
-		}
-	}
-
-	processedFiles := []string{}
-	sort.Strings(packageFiles)
-
-	if forceReplace {
-		list.PrepareIndex()
-	}
-
-	for _, file := range packageFiles {
-		var (
-			stanza deb.Stanza
-			p      *deb.Package
-		)
-
-		candidateProcessedFiles := []string{}
-		isSourcePackage := strings.HasSuffix(file, ".dsc")
-		isUdebPackage := strings.HasSuffix(file, ".udeb")
-
-		if isSourcePackage {
-			stanza, err = deb.GetControlFileFromDsc(file, verifier)
-
-			if err == nil {
-				stanza["Package"] = stanza["Source"]
-				delete(stanza, "Source")
-
-				p, err = deb.NewSourcePackageFromControlFile(stanza)
-			}
-		} else {
-			stanza, err = deb.GetControlFileFromDeb(file)
-			if isUdebPackage {
-				p = deb.NewUdebPackageFromControlFile(stanza)
-			} else {
-				p = deb.NewPackageFromControlFile(stanza)
-			}
-		}
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
-			failedFiles = append(failedFiles, file)
-			continue
-		}
-
-		var checksums utils.ChecksumInfo
-		checksums, err = utils.ChecksumsForFile(file)
-		if err != nil {
-			return err
-		}
-
-		if isSourcePackage {
-			p.UpdateFiles(append(p.Files(), deb.PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
-		} else {
-			p.UpdateFiles([]deb.PackageFile{deb.PackageFile{Filename: filepath.Base(file), Checksums: checksums}})
-		}
-
-		err = context.PackagePool().Import(file, checksums.MD5)
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", file, err)
-			failedFiles = append(failedFiles, file)
-			continue
-		}
-
-		candidateProcessedFiles = append(candidateProcessedFiles, file)
-
-		// go over all files, except for the last one (.dsc/.deb itself)
-		for _, f := range p.Files() {
-			if filepath.Base(f.Filename) == filepath.Base(file) {
-				continue
-			}
-			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
-			err = context.PackagePool().Import(sourceFile, f.Checksums.MD5)
-			if err != nil {
-				context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", sourceFile, err)
-				failedFiles = append(failedFiles, file)
-				break
-			}
-
-			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
-		}
-		if err != nil {
-			// some files haven't been imported
-			continue
-		}
-
-		err = context.CollectionFactory().PackageCollection().Update(p)
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to save package %s: %s@|", p, err)
-			failedFiles = append(failedFiles, file)
-			continue
-		}
-
-		if forceReplace {
-			conflictingPackages := list.Search(deb.Dependency{Pkg: p.Name, Version: p.Version, Architecture: p.Architecture}, true)
-			for _, cp := range conflictingPackages {
-				context.Progress().ColoredPrintf("@r[-]@| %s removed due to conflict with package being added", cp)
-				list.Remove(cp)
-			}
-		}
-
-		err = list.Add(p)
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
-			failedFiles = append(failedFiles, file)
-			continue
-		}
-
-		context.Progress().ColoredPrintf("@g[+]@| %s added@|", p)
-		processedFiles = append(processedFiles, candidateProcessedFiles...)
 	}
 
 	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
@@ -194,7 +60,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save: %s", err)
 	}
 
-	if context.flags.Lookup("remove-files").Value.Get().(bool) {
+	if context.Flags().Lookup("remove-files").Value.Get().(bool) {
 		processedFiles = utils.StrSliceDeduplicate(processedFiles)
 
 		for _, file := range processedFiles {

cmd/repo_create.go

@@ -14,9 +14,17 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	repo := deb.NewLocalRepo(args[0], context.flags.Lookup("comment").Value.String())
+	repo := deb.NewLocalRepo(args[0], context.Flags().Lookup("comment").Value.String())
-	repo.DefaultDistribution = context.flags.Lookup("distribution").Value.String()
+	repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
-	repo.DefaultComponent = context.flags.Lookup("component").Value.String()
+	repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
+
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		repo.Uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+	}
 
 	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
 	if err != nil {
@@ -47,6 +55,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "main", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_drop.go

@@ -34,7 +34,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: local repo is published")
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
 		snapshots := context.CollectionFactory().SnapshotCollection().ByLocalRepoSource(repo)
 

cmd/repo_edit.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/AlekSi/pointer"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -23,16 +25,30 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	if context.flags.Lookup("comment").Value.String() != "" {
+	var uploadersFile *string
-		repo.Comment = context.flags.Lookup("comment").Value.String()
-	}
 
-	if context.flags.Lookup("distribution").Value.String() != "" {
+	context.Flags().Visit(func(flag *flag.Flag) {
-		repo.DefaultDistribution = context.flags.Lookup("distribution").Value.String()
+		switch flag.Name {
-	}
+		case "comment":
+			repo.Comment = flag.Value.String()
+		case "distribution":
+			repo.DefaultDistribution = flag.Value.String()
+		case "component":
+			repo.DefaultComponent = flag.Value.String()
+		case "uploaders-file":
+			uploadersFile = pointer.ToString(flag.Value.String())
+		}
+	})
 
-	if context.flags.Lookup("component").Value.String() != "" {
+	if uploadersFile != nil {
-		repo.DefaultComponent = context.flags.Lookup("component").Value.String()
+		if *uploadersFile != "" {
+			repo.Uploaders, err = deb.NewUploadersFromFile(*uploadersFile)
+			if err != nil {
+				return err
+			}
+		} else {
+			repo.Uploaders = nil
+		}
 	}
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
@@ -63,6 +79,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_include.go

@@ -0,0 +1,234 @@
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"os"
+	"path/filepath"
+	"text/template"
+)
+
+func aptlyRepoInclude(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	verifier, err := getVerifier(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+	}
+
+	if verifier == nil {
+		verifier = &utils.GpgVerifier{}
+	}
+
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
+	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
+	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
+
+	repoTemplate, err := template.New("repo").Parse(context.Flags().Lookup("repo").Value.Get().(string))
+	if err != nil {
+		return fmt.Errorf("error parsing -repo template: %s", err)
+	}
+
+	uploaders := (*deb.Uploaders)(nil)
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+
+		for i := range uploaders.Rules {
+			uploaders.Rules[i].CompiledCondition, err = query.Parse(uploaders.Rules[i].Condition)
+			if err != nil {
+				return fmt.Errorf("error parsing query %s: %s", uploaders.Rules[i].Condition, err)
+			}
+		}
+	}
+
+	reporter := &aptly.ConsoleResultReporter{Progress: context.Progress()}
+
+	var changesFiles, failedFiles, processedFiles []string
+
+	changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)
+
+	for _, path := range changesFiles {
+		var changes *deb.Changes
+
+		changes, err = deb.NewChanges(path)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", path, err)
+			continue
+		}
+
+		err = changes.VerifyAndParse(acceptUnsigned, ignoreSignatures, verifier)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		err = changes.Prepare()
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		repoName := &bytes.Buffer{}
+		err = repoTemplate.Execute(repoName, changes.Stanza)
+		if err != nil {
+			return fmt.Errorf("error applying template to repo: %s", err)
+		}
+
+		context.Progress().Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
+
+		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(repoName.String())
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		currentUploaders := uploaders
+		if repo.Uploaders != nil {
+			currentUploaders = repo.Uploaders
+			for i := range currentUploaders.Rules {
+				currentUploaders.Rules[i].CompiledCondition, err = query.Parse(currentUploaders.Rules[i].Condition)
+				if err != nil {
+					return fmt.Errorf("error parsing query %s: %s", currentUploaders.Rules[i].Condition, err)
+				}
+			}
+		}
+
+		if currentUploaders != nil {
+			if err = currentUploaders.IsAllowed(changes); err != nil {
+				failedFiles = append(failedFiles, path)
+				reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
+					changes.ChangesName, changes.SignatureKeys, err)
+				changes.Cleanup()
+				continue
+			}
+		}
+
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to load repo: %s", err)
+		}
+
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to load packages: %s", err)
+		}
+
+		packageFiles, _ := deb.CollectPackageFiles([]string{changes.TempDir}, reporter)
+
+		var restriction deb.PackageQuery
+
+		restriction, err = changes.PackageQuery()
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		var processedFiles2, failedFiles2 []string
+
+		processedFiles2, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+			context.CollectionFactory().PackageCollection(), reporter, restriction)
+
+		if err != nil {
+			return fmt.Errorf("unable to import package files: %s", err)
+		}
+
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+		if err != nil {
+			return fmt.Errorf("unable to save: %s", err)
+		}
+
+		err = changes.Cleanup()
+		if err != nil {
+			return err
+		}
+
+		for _, file := range failedFiles2 {
+			failedFiles = append(failedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		for _, file := range processedFiles2 {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		processedFiles = append(processedFiles, path)
+	}
+
+	if !noRemoveFiles {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err := os.Remove(file)
+			if err != nil {
+				return fmt.Errorf("unable to remove file: %s", err)
+			}
+		}
+	}
+
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
+	return err
+}
+
+func makeCmdRepoInclude() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoInclude,
+		UsageLine: "include <file.changes>|<directory> ...",
+		Short:     "add packages to local repositories based on .changes files",
+		Long: `
+Command include looks for .changes files in list of arguments or specified directories. Each
+.changes file is verified, parsed, referenced files are put into separate temporary directory
+and added into local repository. Successfully imported files are removed by default.
+
+Additionally uploads could be restricted with <uploaders.json> file. Rules in this file control
+uploads based on GPG key ID of .changes file signature and queries on .changes file fields.
+
+Example:
+
+  $ aptly repo include -repo=foo-release incoming/
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-include", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("no-remove-files", false, "don't remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
+	cmd.Flag.String("repo", "{{.Distribution}}", "which repo should files go to, defaults to Distribution field of .changes file")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of .changes file signature")
+	cmd.Flag.Bool("accept-unsigned", false, "accept unsigned .changes files")
+	cmd.Flag.String("uploaders-file", "", "path to uploaders.json file")
+
+	return cmd
+}

cmd/repo_list.go

@@ -33,6 +33,8 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
+	context.CloseDatabase()
+
 	sort.Strings(repos)
 
 	if raw {

cmd/repo_move.go

@@ -87,7 +87,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	var architecturesList []string
 
-	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
 
 	if withDeps {
 		dstList.PrepareIndex()
@@ -145,7 +145,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
 		dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))

cmd/repo_remove.go

@@ -54,7 +54,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
-	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
 		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))

cmd/repo_search.go

@@ -21,6 +21,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/repo_show.go

@@ -29,9 +29,12 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Comment: %s\n", repo.Comment)
 	fmt.Printf("Default Distribution: %s\n", repo.DefaultDistribution)
 	fmt.Printf("Default Component: %s\n", repo.DefaultComponent)
+	if repo.Uploaders != nil {
+		fmt.Printf("Uploaders: %s\n", repo.Uploaders)
+	}
 	fmt.Printf("Number of packages: %d\n", repo.NumPackages())
 
-	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		ListPackagesRefList(repo.RefList())
 	}

cmd/run.go

@@ -2,18 +2,20 @@ package cmd
 
 import (
 	"fmt"
+	ctx "github.com/smira/aptly/context"
 	"github.com/smira/commander"
+	"os"
 )
 
 // Run runs single command starting from root cmd with args, optionally initializing context
 func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode int) {
 	defer func() {
 		if r := recover(); r != nil {
-			fatal, ok := r.(*FatalError)
+			fatal, ok := r.(*ctx.FatalError)
 			if !ok {
 				panic(r)
 			}
-			fmt.Println("ERROR:", fatal.Message)
+			fmt.Fprintln(os.Stderr, "ERROR:", fatal.Message)
 			returnCode = fatal.ReturnCode
 		}
 	}()
@@ -22,13 +24,13 @@ func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode
 
 	flags, args, err := cmd.ParseFlags(cmdArgs)
 	if err != nil {
-		Fatal(err)
+		ctx.Fatal(err)
 	}
 
 	if initContext {
 		err = InitContext(flags)
 		if err != nil {
-			Fatal(err)
+			ctx.Fatal(err)
 		}
 		defer ShutdownContext()
 	}
@@ -37,7 +39,7 @@ func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode
 
 	err = cmd.Dispatch(args)
 	if err != nil {
-		Fatal(err)
+		ctx.Fatal(err)
 	}
 
 	return

cmd/serve.go

@@ -27,7 +27,7 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		return nil
 	}
 
-	listen := context.flags.Lookup("listen").Value.String()
+	listen := context.Flags().Lookup("listen").Value.String()
 
 	listenHost, listenPort, err := net.SplitHostPort(listen)
 

cmd/snapshot_diff.go

@@ -13,7 +13,7 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	onlyMatching := context.flags.Lookup("only-matching").Value.Get().(bool)
+	onlyMatching := context.Flags().Lookup("only-matching").Value.Get().(bool)
 
 	// Load <name-a> snapshot
 	snapshotA, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])

cmd/snapshot_drop.go

@@ -35,7 +35,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: snapshot is published")
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
 		snapshots := context.CollectionFactory().SnapshotCollection().BySnapshotSource(snapshot)
 		if len(snapshots) > 0 {

cmd/snapshot_filter.go

@@ -17,7 +17,7 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
 
 	// Load <source> snapshot
 	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])

cmd/snapshot_list.go

@@ -4,49 +4,8 @@ import (
 	"fmt"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
-	"sort"
 )
 
-// Snapshot sorting methods
-const (
-	SortName = iota
-	SortTime
-)
-
-type snapshotListToSort struct {
-	list       []*deb.Snapshot
-	sortMethod int
-}
-
-func parseSortMethod(sortMethod string) (int, error) {
-	switch sortMethod {
-	case "time", "Time":
-		return SortTime, nil
-	case "name", "Name":
-		return SortName, nil
-	}
-
-	return -1, fmt.Errorf("sorting method \"%s\" unknown", sortMethod)
-}
-
-func (s snapshotListToSort) Swap(i, j int) {
-	s.list[i], s.list[j] = s.list[j], s.list[i]
-}
-
-func (s snapshotListToSort) Less(i, j int) bool {
-	switch s.sortMethod {
-	case SortName:
-		return s.list[i].Name < s.list[j].Name
-	case SortTime:
-		return s.list[i].CreatedAt.Before(s.list[j].CreatedAt)
-	}
-	panic("unknown sort method")
-}
-
-func (s snapshotListToSort) Len() int {
-	return len(s.list)
-}
-
 func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
@@ -57,33 +16,24 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
 
-	snapshotsToSort := &snapshotListToSort{}
+	collection := context.CollectionFactory().SnapshotCollection()
-	snapshotsToSort.list = make([]*deb.Snapshot, context.CollectionFactory().SnapshotCollection().Len())
-	snapshotsToSort.sortMethod, err = parseSortMethod(sortMethodString)
-	if err != nil {
-		return err
-	}
-
-	i := 0
-	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		snapshotsToSort.list[i] = snapshot
-		i++
-
-		return nil
-	})
-
-	sort.Sort(snapshotsToSort)
 
 	if raw {
-		for _, snapshot := range snapshotsToSort.list {
+		collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
 			fmt.Printf("%s\n", snapshot.Name)
-		}
+			return nil
+		})
 	} else {
-		if len(snapshotsToSort.list) > 0 {
+		if collection.Len() > 0 {
 			fmt.Printf("List of snapshots:\n")
 
-			for _, snapshot := range snapshotsToSort.list {
+			err = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
 				fmt.Printf(" * %s\n", snapshot.String())
+				return nil
+			})
+
+			if err != nil {
+				return err
 			}
 
 			fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
@@ -91,8 +41,8 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
 			fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n")
 		}
 	}
-	return err
 
+	return err
 }
 
 func makeCmdSnapshotList() *commander.Command {

cmd/snapshot_merge.go

@@ -28,22 +28,22 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	latest := context.flags.Lookup("latest").Value.Get().(bool)
+	latest := context.Flags().Lookup("latest").Value.Get().(bool)
-    noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
+	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
 
-    if noRemove && latest {
+	if noRemove && latest {
-        return fmt.Errorf("-no-remove and -latest can't be specified together")
+		return fmt.Errorf("-no-remove and -latest can't be specified together")
-    }
+	}
 
 	overrideMatching := !latest && !noRemove
 
 	result := sources[0].RefList()
 	for i := 1; i < len(sources); i++ {
-		result = result.Merge(sources[i].RefList(), overrideMatching)
+		result = result.Merge(sources[i].RefList(), overrideMatching, false)
 	}
 
 	if latest {
-		deb.FilterLatestRefs(result)
+		result.FilterLatestRefs()
 	}
 
 	sourceDescription := make([]string, len(sources))
@@ -84,7 +84,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("latest", false, "use only the latest version of each package")
-    cmd.Flag.Bool("no-remove", false, "don't remove duplicate arch/name packages")
+	cmd.Flag.Bool("no-remove", false, "don't remove duplicate arch/name packages")
 
 	return cmd
 }

cmd/snapshot_pull.go

@@ -17,9 +17,9 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	noDeps := context.flags.Lookup("no-deps").Value.Get().(bool)
+	noDeps := context.Flags().Lookup("no-deps").Value.Get().(bool)
-	noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
+	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
-	allMatches := context.flags.Lookup("all-matches").Value.Get().(bool)
+	allMatches := context.Flags().Lookup("all-matches").Value.Get().(bool)
 
 	// Load <name> snapshot
 	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
@@ -91,7 +91,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to parse query: %s", err)
 		}
 		// Add architecture filter
-		queries[i] = &deb.AndQuery{queries[i], archQuery}
+		queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
 	}
 
 	// Filter with dependencies as requested
@@ -129,7 +129,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	})
 	alreadySeen = nil
 
-	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nNot creating snapshot, as dry run was requested.\n")
 	} else {
 		// Create <destination> snapshot

cmd/snapshot_search.go

@@ -73,7 +73,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		return fmt.Errorf("unable to search: %s", err)
 	}
 
-	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
 	architecturesList := []string{}
 
 	if withDeps {
@@ -96,10 +96,12 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		return fmt.Errorf("unable to search: %s", err)
 	}
 
-	result.ForEach(func(p *deb.Package) error {
+	if result.Len() == 0 {
-		context.Progress().Printf("%s\n", p)
+		return fmt.Errorf("no results")
-		return nil
+	}
-	})
+
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format)
 
 	return err
 }
@@ -120,6 +122,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/snapshot_show.go

@@ -30,7 +30,7 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Description: %s\n", snapshot.Description)
 	fmt.Printf("Number of packages: %d\n", snapshot.NumPackages())
 
-	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		ListPackagesRefList(snapshot.RefList())
 	}

cmd/snapshot_verify.go

@@ -31,25 +31,25 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 
 	packageList, err := deb.NewPackageListFromRefList(snapshots[0].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
-		fmt.Errorf("unable to load packages: %s", err)
+		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
 	sourcePackageList := deb.NewPackageList()
 	err = sourcePackageList.Append(packageList)
 	if err != nil {
-		fmt.Errorf("unable to merge sources: %s", err)
+		return fmt.Errorf("unable to merge sources: %s", err)
 	}
 
 	var pL *deb.PackageList
 	for i := 1; i < len(snapshots); i++ {
 		pL, err = deb.NewPackageListFromRefList(snapshots[i].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 		if err != nil {
-			fmt.Errorf("unable to load packages: %s", err)
+			return fmt.Errorf("unable to load packages: %s", err)
 		}
 
 		err = sourcePackageList.Append(pL)
 		if err != nil {
-			fmt.Errorf("unable to merge sources: %s", err)
+			return fmt.Errorf("unable to merge sources: %s", err)
 		}
 	}
 

cmd/task_run.go

@@ -18,13 +18,15 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 		var text string
 		cmdArgs := []string{}
 
-		if finfo, err := os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
+		var finfo os.FileInfo
+		if finfo, err = os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
 			return fmt.Errorf("no such file, %s\n", filename)
 		}
 
-		fmt.Println("Reading file...\n")
+		fmt.Print("Reading file...\n\n")
 
-		file, err := os.Open(filename)
+		var file *os.File
+		file, err = os.Open(filename)
 
 		if err != nil {
 			return err
@@ -80,6 +82,10 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 
 	for i, command := range cmdList {
 		if !commandErrored {
+			err := context.ReOpenDatabase()
+			if err != nil {
+				return fmt.Errorf("failed to reopen DB: %s", err)
+			}
 			context.Progress().ColoredPrintf("@g%d) [Running]: %s@!", (i + 1), strings.Join(command, " "))
 			context.Progress().ColoredPrintf("\n@yBegin command output: ----------------------------@!")
 			context.Progress().Flush()
@@ -129,16 +135,16 @@ func makeCmdTaskRun() *commander.Command {
 		UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
 		Short:     "run aptly tasks",
 		Long: `
-Command helps origanise multiple aptly commands in one single aptly task, running as single thread.
+Command helps organise multiple aptly commands in one single aptly task, running as single thread.
 
 Example:
 
-  $ aptly task run
+	  $ aptly task run
-  > repo create local
+	  > repo create local
-  > repo add local pkg1
+	  > repo add local pkg1
-  > publish repo local
+	  > publish repo local
-  > serve
+	  > serve
-  >
+	  >
 
 `,
 	}

console/progress.go

@@ -14,6 +14,8 @@ const (
 	codeHideProgress
 	codeStop
 	codeFlush
+	codeBarEnabled
+	codeBarDisabled
 )
 
 type printTask struct {
@@ -81,6 +83,8 @@ func (p *Progress) InitBar(count int64, isBytes bool) {
 			p.bar.SetUnits(pb.U_BYTES)
 			p.bar.ShowSpeed = true
 		}
+
+		p.queue <- printTask{code: codeBarEnabled}
 		p.bar.Start()
 	}
 }
@@ -91,6 +95,7 @@ func (p *Progress) ShutdownBar() {
 		return
 	}
 	p.bar.Finish()
+	p.queue <- printTask{code: codeBarDisabled}
 	p.bar = nil
 	p.queue <- printTask{code: codeHideProgress}
 }
@@ -128,19 +133,30 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
 		p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"}
 	} else {
 		// stip color marks
-		var prev rune
+		var inColorMark, inCurly bool
 		msg = strings.Map(func(r rune) rune {
-			if prev == '@' {
+			if inColorMark {
-				prev = 0
+				if inCurly {
-				if r == '@' {
+					if r == '}' {
-					return r
+						inCurly = false
+						inColorMark = false
+						return -1
+					}
+				} else {
+					if r == '{' {
+						inCurly = true
+					} else if r == '@' {
+						return '@'
+					} else {
+						inColorMark = false
+					}
 				}
 				return -1
 			}
-			prev = r
-			if r == '@' {
-				return -1
 
+			if r == '@' {
+				inColorMark = true
+				return -1
 			}
 
 			return r
@@ -151,9 +167,15 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
 }
 
 func (p *Progress) worker() {
+	hasBar := false
+
 	for {
 		task := <-p.queue
 		switch task.code {
+		case codeBarEnabled:
+			hasBar = true
+		case codeBarDisabled:
+			hasBar = false
 		case codePrint:
 			if p.barShown {
 				fmt.Print("\r\033[2K")
@@ -161,7 +183,7 @@ func (p *Progress) worker() {
 			}
 			fmt.Print(task.message)
 		case codeProgress:
-			if p.bar != nil {
+			if hasBar {
 				fmt.Print("\r" + task.message)
 				p.barShown = true
 			}

console/terminal.go

@@ -1,9 +1,7 @@
-// +build !freebsd
-
 package console
 
 import (
-	"code.google.com/p/go.crypto/ssh/terminal"
+	"golang.org/x/crypto/ssh/terminal"
 	"syscall"
 )
 

console/terminal_bsd.go

@@ -1,10 +0,0 @@
-// +build freebsd
-
-package console
-
-// RunningOnTerminal checks whether stdout is terminal
-//
-// Stub for FreeBSD, until in go1.3 terminal.IsTerminal would start working for FreeBSD
-func RunningOnTerminal() bool {
-	return false
-}

context/context.go

@@ -0,0 +1,492 @@
+// Package context provides single entry to all resources
+package context
+
+import (
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/console"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/files"
+	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/s3"
+	"github.com/smira/aptly/swift"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"strings"
+	"sync"
+	"time"
+)
+
+// AptlyContext is a common context shared by all commands
+type AptlyContext struct {
+	sync.Mutex
+
+	flags, globalFlags *flag.FlagSet
+	configLoaded       bool
+
+	progress          aptly.Progress
+	downloader        aptly.Downloader
+	database          database.Storage
+	packagePool       aptly.PackagePool
+	publishedStorages map[string]aptly.PublishedStorage
+	collectionFactory *deb.CollectionFactory
+	dependencyOptions int
+	architecturesList []string
+	// Debug features
+	fileCPUProfile *os.File
+	fileMemProfile *os.File
+	fileMemStats   *os.File
+}
+
+// Check interface
+var _ aptly.PublishedStorageProvider = &AptlyContext{}
+
+// FatalError is type for panicking to abort execution with non-zero
+// exit code and print meaningful explanation
+type FatalError struct {
+	ReturnCode int
+	Message    string
+}
+
+// Fatal panics and aborts execution with exit code 1
+func Fatal(err error) {
+	returnCode := 1
+	if err == commander.ErrFlagError || err == commander.ErrCommandError {
+		returnCode = 2
+	}
+	panic(&FatalError{ReturnCode: returnCode, Message: err.Error()})
+}
+
+// Config loads and returns current configuration
+func (context *AptlyContext) Config() *utils.ConfigStructure {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.config()
+}
+
+func (context *AptlyContext) config() *utils.ConfigStructure {
+	if !context.configLoaded {
+		var err error
+
+		configLocation := context.globalFlags.Lookup("config").Value.String()
+		if configLocation != "" {
+			err = utils.LoadConfig(configLocation, &utils.Config)
+
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			configLocations := []string{
+				filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
+				"/etc/aptly.conf",
+			}
+
+			for _, configLocation := range configLocations {
+				err = utils.LoadConfig(configLocation, &utils.Config)
+				if err == nil {
+					break
+				}
+				if !os.IsNotExist(err) {
+					Fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
+				}
+			}
+
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", configLocations[0])
+				utils.SaveConfig(configLocations[0], &utils.Config)
+			}
+		}
+
+		context.configLoaded = true
+
+	}
+	return &utils.Config
+}
+
+// LookupOption checks boolean flag with default (usually config) and command-line
+// setting
+func (context *AptlyContext) LookupOption(defaultValue bool, name string) (result bool) {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.lookupOption(defaultValue, name)
+}
+
+func (context *AptlyContext) lookupOption(defaultValue bool, name string) (result bool) {
+	result = defaultValue
+
+	if context.globalFlags.IsSet(name) {
+		result = context.globalFlags.Lookup(name).Value.Get().(bool)
+	}
+
+	return
+}
+
+// DependencyOptions calculates options related to dependecy handling
+func (context *AptlyContext) DependencyOptions() int {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.dependencyOptions == -1 {
+		context.dependencyOptions = 0
+		if context.lookupOption(context.config().DepFollowSuggests, "dep-follow-suggests") {
+			context.dependencyOptions |= deb.DepFollowSuggests
+		}
+		if context.lookupOption(context.config().DepFollowRecommends, "dep-follow-recommends") {
+			context.dependencyOptions |= deb.DepFollowRecommends
+		}
+		if context.lookupOption(context.config().DepFollowAllVariants, "dep-follow-all-variants") {
+			context.dependencyOptions |= deb.DepFollowAllVariants
+		}
+		if context.lookupOption(context.config().DepFollowSource, "dep-follow-source") {
+			context.dependencyOptions |= deb.DepFollowSource
+		}
+	}
+
+	return context.dependencyOptions
+}
+
+// ArchitecturesList returns list of architectures fixed via command line or config
+func (context *AptlyContext) ArchitecturesList() []string {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.architecturesList == nil {
+		context.architecturesList = context.config().Architectures
+		optionArchitectures := context.globalFlags.Lookup("architectures").Value.String()
+		if optionArchitectures != "" {
+			context.architecturesList = strings.Split(optionArchitectures, ",")
+		}
+	}
+
+	return context.architecturesList
+}
+
+// Progress creates or returns Progress object
+func (context *AptlyContext) Progress() aptly.Progress {
+	context.Lock()
+	defer context.Unlock()
+
+	return context._progress()
+}
+
+func (context *AptlyContext) _progress() aptly.Progress {
+	if context.progress == nil {
+		context.progress = console.NewProgress()
+		context.progress.Start()
+	}
+
+	return context.progress
+}
+
+// Downloader returns instance of current downloader
+func (context *AptlyContext) Downloader() aptly.Downloader {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.downloader == nil {
+		var downloadLimit int64
+		limitFlag := context.flags.Lookup("download-limit")
+		if limitFlag != nil {
+			downloadLimit = limitFlag.Value.Get().(int64)
+		}
+		if downloadLimit == 0 {
+			downloadLimit = context.config().DownloadLimit
+		}
+		context.downloader = http.NewDownloader(context.config().DownloadConcurrency,
+			downloadLimit*1024, context._progress())
+	}
+
+	return context.downloader
+}
+
+// DBPath builds path to database
+func (context *AptlyContext) DBPath() string {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.dbPath()
+}
+
+// DBPath builds path to database
+func (context *AptlyContext) dbPath() string {
+	return filepath.Join(context.config().RootDir, "db")
+}
+
+// Database opens and returns current instance of database
+func (context *AptlyContext) Database() (database.Storage, error) {
+	context.Lock()
+	defer context.Unlock()
+
+	return context._database()
+}
+
+func (context *AptlyContext) _database() (database.Storage, error) {
+	if context.database == nil {
+		var err error
+
+		context.database, err = database.OpenDB(context.dbPath())
+		if err != nil {
+			return nil, fmt.Errorf("can't open database: %s", err)
+		}
+	}
+
+	return context.database, nil
+}
+
+// CloseDatabase closes the db temporarily
+func (context *AptlyContext) CloseDatabase() error {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.database == nil {
+		return nil
+	}
+
+	return context.database.Close()
+}
+
+// ReOpenDatabase reopens the db after close
+func (context *AptlyContext) ReOpenDatabase() error {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.database == nil {
+		return nil
+	}
+
+	const MaxTries = 10
+	const Delay = 10 * time.Second
+
+	for try := 0; try < MaxTries; try++ {
+		err := context.database.ReOpen()
+		if err == nil || strings.Index(err.Error(), "resource temporarily unavailable") == -1 {
+			return err
+		}
+		context._progress().Printf("Unable to reopen database, sleeping %s\n", Delay)
+		<-time.After(Delay)
+	}
+
+	return fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
+}
+
+// CollectionFactory builds factory producing all kinds of collections
+func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.collectionFactory == nil {
+		db, err := context._database()
+		if err != nil {
+			Fatal(err)
+		}
+		context.collectionFactory = deb.NewCollectionFactory(db)
+	}
+
+	return context.collectionFactory
+}
+
+// PackagePool returns instance of PackagePool
+func (context *AptlyContext) PackagePool() aptly.PackagePool {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.packagePool == nil {
+		context.packagePool = files.NewPackagePool(context.config().RootDir)
+	}
+
+	return context.packagePool
+}
+
+// GetPublishedStorage returns instance of PublishedStorage
+func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
+	context.Lock()
+	defer context.Unlock()
+
+	publishedStorage, ok := context.publishedStorages[name]
+	if !ok {
+		if name == "" {
+			publishedStorage = files.NewPublishedStorage(context.config().RootDir)
+		} else if strings.HasPrefix(name, "s3:") {
+			params, ok := context.config().S3PublishRoots[name[3:]]
+			if !ok {
+				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
+			}
+
+			var err error
+			publishedStorage, err = s3.NewPublishedStorage(
+				params.AccessKeyID, params.SecretAccessKey, params.SessionToken,
+				params.Region, params.Endpoint, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
+				params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel,
+				params.ForceSigV2, params.Debug)
+			if err != nil {
+				Fatal(err)
+			}
+		} else if strings.HasPrefix(name, "swift:") {
+			params, ok := context.config().SwiftPublishRoots[name[6:]]
+			if !ok {
+				Fatal(fmt.Errorf("published Swift storage %v not configured", name[6:]))
+			}
+
+			var err error
+			publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password,
+				params.AuthURL, params.Tenant, params.TenantID, params.Container, params.Prefix)
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			Fatal(fmt.Errorf("unknown published storage format: %v", name))
+		}
+		context.publishedStorages[name] = publishedStorage
+	}
+
+	return publishedStorage
+}
+
+// UploadPath builds path to upload storage
+func (context *AptlyContext) UploadPath() string {
+	return filepath.Join(context.Config().RootDir, "upload")
+}
+
+// UpdateFlags sets internal copy of flags in the context
+func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) {
+	context.Lock()
+	defer context.Unlock()
+
+	context.flags = flags
+}
+
+// Flags returns current command flags
+func (context *AptlyContext) Flags() *flag.FlagSet {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.flags
+}
+
+// GlobalFlags returns flags passed to all commands
+func (context *AptlyContext) GlobalFlags() *flag.FlagSet {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.globalFlags
+}
+
+// Shutdown shuts context down
+func (context *AptlyContext) Shutdown() {
+	context.Lock()
+	defer context.Unlock()
+
+	if aptly.EnableDebug {
+		if context.fileMemProfile != nil {
+			pprof.WriteHeapProfile(context.fileMemProfile)
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+		if context.fileCPUProfile != nil {
+			pprof.StopCPUProfile()
+			context.fileCPUProfile.Close()
+			context.fileCPUProfile = nil
+		}
+		if context.fileMemProfile != nil {
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+	}
+	if context.database != nil {
+		context.database.Close()
+		context.database = nil
+	}
+	if context.downloader != nil {
+		context.downloader.Abort()
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
+// Cleanup does partial shutdown of context
+func (context *AptlyContext) Cleanup() {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.downloader != nil {
+		context.downloader.Shutdown()
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
+// NewContext initializes context with default settings
+func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
+	var err error
+
+	context := &AptlyContext{
+		flags:             flags,
+		globalFlags:       flags,
+		dependencyOptions: -1,
+		publishedStorages: map[string]aptly.PublishedStorage{},
+	}
+
+	if aptly.EnableDebug {
+		cpuprofile := flags.Lookup("cpuprofile").Value.String()
+		if cpuprofile != "" {
+			context.fileCPUProfile, err = os.Create(cpuprofile)
+			if err != nil {
+				return nil, err
+			}
+			pprof.StartCPUProfile(context.fileCPUProfile)
+		}
+
+		memprofile := flags.Lookup("memprofile").Value.String()
+		if memprofile != "" {
+			context.fileMemProfile, err = os.Create(memprofile)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		memstats := flags.Lookup("memstats").Value.String()
+		if memstats != "" {
+			interval := flags.Lookup("meminterval").Value.Get().(time.Duration)
+
+			context.fileMemStats, err = os.Create(memstats)
+			if err != nil {
+				return nil, err
+			}
+
+			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
+
+			go func() {
+				var stats runtime.MemStats
+
+				start := time.Now().UnixNano()
+
+				for {
+					runtime.ReadMemStats(&stats)
+					if context.fileMemStats != nil {
+						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
+							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
+						time.Sleep(interval)
+					} else {
+						break
+					}
+				}
+			}()
+		}
+	}
+
+	return context, nil
+}

database/leveldb.go

@@ -43,7 +43,8 @@ var (
 
 func internalOpen(path string) (*leveldb.DB, error) {
 	o := &opt.Options{
-		Filter: filter.NewBloomFilter(10),
+		Filter:                 filter.NewBloomFilter(10),
+		OpenFilesCacheCapacity: 256,
 	}
 
 	return leveldb.OpenFile(path, o)
@@ -60,7 +61,7 @@ func OpenDB(path string) (Storage, error) {
 
 // RecoverDB recovers LevelDB database from corruption
 func RecoverDB(path string) error {
-	stor, err := storage.OpenFile(path)
+	stor, err := storage.OpenFile(path, false)
 	if err != nil {
 		return err
 	}
@@ -101,7 +102,7 @@ func (l *levelDB) Put(key []byte, value []byte) error {
 			return err
 		}
 	} else {
-		if bytes.Compare(old, value) == 0 {
+		if bytes.Equal(old, value) {
 			return nil
 		}
 	}

database/leveldb_test.go

@@ -1,8 +1,9 @@
 package database
 
 import (
-	. "launchpad.net/gocheck"
 	"testing"
+
+	. "gopkg.in/check.v1"
 )
 
 // Launch gocheck tests

deb/changes.go

@@ -0,0 +1,273 @@
+package deb
+
+import (
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// Changes is a result of .changes file parsing
+type Changes struct {
+	Changes               string
+	Distribution          string
+	Files                 PackageFiles
+	BasePath, ChangesName string
+	TempDir               string
+	Source                string
+	Binary                []string
+	Architectures         []string
+	Stanza                Stanza
+	SignatureKeys         []utils.GpgKey
+}
+
+// NewChanges moves .changes file into temporary directory and creates Changes structure
+func NewChanges(path string) (*Changes, error) {
+	var err error
+
+	c := &Changes{
+		BasePath:    filepath.Dir(path),
+		ChangesName: filepath.Base(path),
+	}
+
+	c.TempDir, err = ioutil.TempDir(os.TempDir(), "aptly")
+	if err != nil {
+		return nil, err
+	}
+
+	// copy .changes file into temporary directory
+	err = utils.CopyFile(filepath.Join(c.BasePath, c.ChangesName), filepath.Join(c.TempDir, c.ChangesName))
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// VerifyAndParse does optional signature verification and parses changes files
+func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier utils.Verifier) error {
+	input, err := os.Open(filepath.Join(c.TempDir, c.ChangesName))
+	if err != nil {
+		return err
+	}
+	defer input.Close()
+
+	isClearSigned, err := verifier.IsClearSigned(input)
+	if err != nil {
+		return err
+	}
+
+	input.Seek(0, 0)
+
+	if !isClearSigned && !acceptUnsigned {
+		return fmt.Errorf(".changes file is not signed and unsigned processing hasn't been enabled")
+	}
+
+	if isClearSigned && !ignoreSignature {
+		keyInfo, err := verifier.VerifyClearsigned(input, false)
+		if err != nil {
+			return err
+		}
+		input.Seek(0, 0)
+
+		c.SignatureKeys = keyInfo.GoodKeys
+	}
+
+	var text *os.File
+
+	if isClearSigned {
+		text, err = verifier.ExtractClearsigned(input)
+		if err != nil {
+			return err
+		}
+		defer text.Close()
+	} else {
+		text = input
+	}
+
+	reader := NewControlFileReader(text)
+	c.Stanza, err = reader.ReadStanza(false)
+	if err != nil {
+		return err
+	}
+
+	c.Distribution = c.Stanza["Distribution"]
+	c.Changes = c.Stanza["Changes"]
+	c.Source = c.Stanza["Source"]
+	c.Binary = strings.Fields(c.Stanza["Binary"])
+	c.Architectures = strings.Fields(c.Stanza["Architecture"])
+
+	c.Files, err = c.Files.ParseSumFields(c.Stanza)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Prepare creates temporary directory, copies file there and verifies checksums
+func (c *Changes) Prepare() error {
+	var err error
+
+	for _, file := range c.Files {
+		if filepath.Dir(file.Filename) != "." {
+			return fmt.Errorf("file is not in the same folder as .changes file: %s", file.Filename)
+		}
+
+		file.Filename = filepath.Base(file.Filename)
+
+		err = utils.CopyFile(filepath.Join(c.BasePath, file.Filename), filepath.Join(c.TempDir, file.Filename))
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, file := range c.Files {
+		var info utils.ChecksumInfo
+
+		info, err = utils.ChecksumsForFile(filepath.Join(c.TempDir, file.Filename))
+		if err != nil {
+			return err
+		}
+
+		if info.Size != file.Checksums.Size {
+			return fmt.Errorf("size mismatch: expected %v != obtained %v", file.Checksums.Size, info.Size)
+		}
+
+		if info.MD5 != file.Checksums.MD5 {
+			return fmt.Errorf("checksum mismatch MD5: expected %v != obtained %v", file.Checksums.MD5, info.MD5)
+		}
+
+		if info.SHA1 != file.Checksums.SHA1 {
+			return fmt.Errorf("checksum mismatch SHA1: expected %v != obtained %v", file.Checksums.SHA1, info.SHA1)
+		}
+
+		if info.SHA256 != file.Checksums.SHA256 {
+			return fmt.Errorf("checksum mismatch SHA256 expected %v != obtained %v", file.Checksums.SHA256, info.SHA256)
+		}
+	}
+
+	return nil
+}
+
+// Cleanup removes all temporary files
+func (c *Changes) Cleanup() error {
+	if c.TempDir == "" {
+		return nil
+	}
+
+	return os.RemoveAll(c.TempDir)
+}
+
+// PackageQuery returns query that every package should match to be included
+func (c *Changes) PackageQuery() (PackageQuery, error) {
+	var archQuery PackageQuery = &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: ""}
+	for _, arch := range c.Architectures {
+		archQuery = &OrQuery{L: &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: arch}, R: archQuery}
+	}
+
+	// if c.Source is empty, this would never match
+	sourceQuery := &AndQuery{
+		L: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"},
+		R: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Source},
+	}
+
+	var binaryQuery PackageQuery
+	if len(c.Binary) > 0 {
+		binaryQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Binary[0]}
+		for _, binary := range c.Binary[1:] {
+			binaryQuery = &OrQuery{
+				L: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: binary},
+				R: binaryQuery,
+			}
+		}
+
+		binaryQuery = &AndQuery{
+			L: &NotQuery{Q: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"}},
+			R: binaryQuery}
+	}
+
+	var nameQuery PackageQuery
+	if binaryQuery == nil {
+		nameQuery = sourceQuery
+	} else {
+		nameQuery = &OrQuery{L: sourceQuery, R: binaryQuery}
+	}
+
+	return &AndQuery{L: archQuery, R: nameQuery}, nil
+}
+
+// GetField implements PackageLike interface
+func (c *Changes) GetField(field string) string {
+	return c.Stanza[field]
+}
+
+// MatchesDependency implements PackageLike interface
+func (c *Changes) MatchesDependency(d Dependency) bool {
+	return false
+}
+
+// MatchesArchitecture implements PackageLike interface
+func (c *Changes) MatchesArchitecture(arch string) bool {
+	return false
+}
+
+// GetName implements PackageLike interface
+func (c *Changes) GetName() string {
+	return ""
+}
+
+// GetVersion implements PackageLike interface
+func (c *Changes) GetVersion() string {
+	return ""
+
+}
+
+// GetArchitecture implements PackageLike interface
+func (c *Changes) GetArchitecture() string {
+	return ""
+}
+
+// CollectChangesFiles walks filesystem collecting all .changes files
+func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (changesFiles, failedFiles []string) {
+	for _, location := range locations {
+		info, err2 := os.Stat(location)
+		if err2 != nil {
+			reporter.Warning("Unable to process %s: %s", location, err2)
+			failedFiles = append(failedFiles, location)
+			continue
+		}
+		if info.IsDir() {
+			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
+				if err3 != nil {
+					return err3
+				}
+				if info.IsDir() {
+					return nil
+				}
+
+				if strings.HasSuffix(info.Name(), ".changes") {
+					changesFiles = append(changesFiles, path)
+				}
+
+				return nil
+			})
+
+			if err2 != nil {
+				reporter.Warning("Unable to process %s: %s", location, err2)
+				failedFiles = append(failedFiles, location)
+				continue
+			}
+		} else if strings.HasSuffix(info.Name(), ".changes") {
+			changesFiles = append(changesFiles, location)
+		}
+	}
+
+	sort.Strings(changesFiles)
+
+	return
+}

deb/changes_test.go

@@ -0,0 +1,91 @@
+package deb
+
+import (
+	. "gopkg.in/check.v1"
+	"os"
+	"path/filepath"
+)
+
+type ChangesSuite struct {
+	Dir, Path string
+}
+
+var _ = Suite(&ChangesSuite{})
+
+func (s *ChangesSuite) SetUpTest(c *C) {
+	s.Dir = c.MkDir()
+	s.Path = filepath.Join(s.Dir, "calamares.changes")
+
+	f, err := os.Create(s.Path)
+	c.Assert(err, IsNil)
+
+	f.WriteString(changesFile)
+	f.Close()
+}
+
+func (s *ChangesSuite) TestParseAndVerify(c *C) {
+	changes, err := NewChanges(s.Path)
+	c.Assert(err, IsNil)
+
+	err = changes.VerifyAndParse(true, true, &NullVerifier{})
+	c.Check(err, IsNil)
+
+	c.Check(changes.Distribution, Equals, "sid")
+	c.Check(changes.Files, HasLen, 4)
+	c.Check(changes.Files[0].Filename, Equals, "calamares_0+git20141127.99.dsc")
+	c.Check(changes.Files[0].Checksums.Size, Equals, int64(1106))
+	c.Check(changes.Files[0].Checksums.MD5, Equals, "05fd8f3ffe8f362c5ef9bad2f936a56e")
+	c.Check(changes.Files[0].Checksums.SHA1, Equals, "79f10e955dab6eb25b7f7bae18213f367a3a0396")
+	c.Check(changes.Files[0].Checksums.SHA256, Equals, "35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc")
+	c.Check(changes.BasePath, Equals, s.Dir)
+	c.Check(changes.Architectures, DeepEquals, []string{"source", "amd64"})
+	c.Check(changes.Source, Equals, "calamares")
+	c.Check(changes.Binary, DeepEquals, []string{"calamares", "calamares-dbg"})
+}
+
+func (s *ChangesSuite) TestPackageQuery(c *C) {
+	changes, err := NewChanges(s.Path)
+	c.Assert(err, IsNil)
+
+	err = changes.VerifyAndParse(true, true, &NullVerifier{})
+	c.Check(err, IsNil)
+
+	q, err := changes.PackageQuery()
+	c.Check(err, IsNil)
+
+	c.Check(q.String(), Equals,
+		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), ((Name (= calamares-dbg)) | (Name (= calamares)))))")
+}
+
+var changesFile = `Format: 1.8
+Date: Thu, 27 Nov 2014 13:24:53 +0000
+Source: calamares
+Binary: calamares calamares-dbg
+Architecture: source amd64
+Version: 0+git20141127.99
+Distribution: sid
+Urgency: medium
+Maintainer: Rohan Garg <rohan@kde.org>
+Changed-By: Rohan <rohan@kde.org>
+Description:
+ calamares  - distribution-independent installer framework
+ calamares-dbg - distribution-independent installer framework -- debug symbols
+Changes:
+ calamares (0+git20141127.99) sid; urgency=medium
+ .
+   * Update from git
+Checksums-Sha1:
+ 79f10e955dab6eb25b7f7bae18213f367a3a0396 1106 calamares_0+git20141127.99.dsc
+ 294c28e2c8e34e72ca9ee0d9da5c14f3bf4188db 2694800 calamares_0+git20141127.99.tar.xz
+ d6c26c04b5407c7511f61cb3e3de60c4a1d6c4ff 1698924 calamares_0+git20141127.99_amd64.deb
+ a3da632d193007b0d4a1aff73159fde1b532d7a8 12835902 calamares-dbg_0+git20141127.99_amd64.deb
+Checksums-Sha256:
+ 35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc 1106 calamares_0+git20141127.99.dsc
+ 5576b9caaf814564830f95561227e4f04ee87b31da22c1371aab155cbf7ce395 2694800 calamares_0+git20141127.99.tar.xz
+ 2e6e2f232ed7ffe52369928ebdf5436d90feb37840286ffba79e87d57a43a2e9 1698924 calamares_0+git20141127.99_amd64.deb
+ 8dd926080ed7bad2e2439e37e49ce12d5f1357c5041b7da4d860a1041f878a8a 12835902 calamares-dbg_0+git20141127.99_amd64.deb
+Files:
+ 05fd8f3ffe8f362c5ef9bad2f936a56e 1106 devel optional calamares_0+git20141127.99.dsc
+ 097e55c81abd8e5f30bb2eed90c2c1e9 2694800 devel optional calamares_0+git20141127.99.tar.xz
+ 827fb3b12534241e119815d331e8197b 1698924 devel optional calamares_0+git20141127.99_amd64.deb
+ e6f8ce70f564d1f68cb57758b15b13e3 12835902 debug optional calamares-dbg_0+git20141127.99_amd64.deb`

deb/collections.go

@@ -2,10 +2,12 @@ package deb
 
 import (
 	"github.com/smira/aptly/database"
+	"sync"
 )
 
 // CollectionFactory is a single place to generate all desired collections
 type CollectionFactory struct {
+	*sync.Mutex
 	db             database.Storage
 	packages       *PackageCollection
 	remoteRepos    *RemoteRepoCollection
@@ -16,11 +18,14 @@ type CollectionFactory struct {
 
 // NewCollectionFactory creates new factory
 func NewCollectionFactory(db database.Storage) *CollectionFactory {
-	return &CollectionFactory{db: db}
+	return &CollectionFactory{Mutex: &sync.Mutex{}, db: db}
 }
 
 // PackageCollection returns (or creates) new PackageCollection
 func (factory *CollectionFactory) PackageCollection() *PackageCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
 	if factory.packages == nil {
 		factory.packages = NewPackageCollection(factory.db)
 	}
@@ -30,6 +35,9 @@ func (factory *CollectionFactory) PackageCollection() *PackageCollection {
 
 // RemoteRepoCollection returns (or creates) new RemoteRepoCollection
 func (factory *CollectionFactory) RemoteRepoCollection() *RemoteRepoCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
 	if factory.remoteRepos == nil {
 		factory.remoteRepos = NewRemoteRepoCollection(factory.db)
 	}
@@ -39,6 +47,9 @@ func (factory *CollectionFactory) RemoteRepoCollection() *RemoteRepoCollection {
 
 // SnapshotCollection returns (or creates) new SnapshotCollection
 func (factory *CollectionFactory) SnapshotCollection() *SnapshotCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
 	if factory.snapshots == nil {
 		factory.snapshots = NewSnapshotCollection(factory.db)
 	}
@@ -48,6 +59,9 @@ func (factory *CollectionFactory) SnapshotCollection() *SnapshotCollection {
 
 // LocalRepoCollection returns (or creates) new LocalRepoCollection
 func (factory *CollectionFactory) LocalRepoCollection() *LocalRepoCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
 	if factory.localRepos == nil {
 		factory.localRepos = NewLocalRepoCollection(factory.db)
 	}
@@ -57,9 +71,24 @@ func (factory *CollectionFactory) LocalRepoCollection() *LocalRepoCollection {
 
 // PublishedRepoCollection returns (or creates) new PublishedRepoCollection
 func (factory *CollectionFactory) PublishedRepoCollection() *PublishedRepoCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
 	if factory.publishedRepos == nil {
 		factory.publishedRepos = NewPublishedRepoCollection(factory.db)
 	}
 
 	return factory.publishedRepos
 }
+
+// Flush removes all references to collections, so that memory could be reclaimed
+func (factory *CollectionFactory) Flush() {
+	factory.Lock()
+	defer factory.Unlock()
+
+	factory.localRepos = nil
+	factory.snapshots = nil
+	factory.remoteRepos = nil
+	factory.publishedRepos = nil
+	factory.packages = nil
+}

deb/contents.go

@@ -0,0 +1,75 @@
+package deb
+
+import (
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"io"
+	"sort"
+	"strings"
+)
+
+// ContentsIndex calculates mapping from files to packages, with sorting and aggregation
+type ContentsIndex struct {
+	index map[string][]*Package
+}
+
+// NewContentsIndex creates empty ContentsIndex
+func NewContentsIndex() *ContentsIndex {
+	return &ContentsIndex{
+		index: make(map[string][]*Package),
+	}
+}
+
+// Push adds package to contents index, calculating package contents as required
+func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool) {
+	contents := p.Contents(packagePool)
+
+	for _, path := range contents {
+		index.index[path] = append(index.index[path], p)
+	}
+}
+
+// Empty checks whether index contains no packages
+func (index *ContentsIndex) Empty() bool {
+	return len(index.index) == 0
+}
+
+// WriteTo dumps sorted mapping of files to qualified package names
+func (index *ContentsIndex) WriteTo(w io.Writer) (int64, error) {
+	var n int64
+
+	paths := make([]string, len(index.index))
+
+	i := 0
+	for path := range index.index {
+		paths[i] = path
+		i++
+	}
+
+	sort.Strings(paths)
+
+	nn, err := fmt.Fprintf(w, "%s %s\n", "FILE", "LOCATION")
+	n += int64(nn)
+	if err != nil {
+		return n, err
+	}
+
+	for _, path := range paths {
+		packages := index.index[path]
+		parts := make([]string, 0, len(packages))
+		for i := range packages {
+			name := packages[i].QualifiedName()
+			if !utils.StrSliceHasItem(parts, name) {
+				parts = append(parts, name)
+			}
+		}
+		nn, err = fmt.Fprintf(w, "%s %s\n", path, strings.Join(parts, ","))
+		n += int64(nn)
+		if err != nil {
+			return n, err
+		}
+	}
+
+	return n, nil
+}

deb/deb.go

@@ -2,11 +2,13 @@ package deb
 
 import (
 	"archive/tar"
-	"bufio"
+	"compress/bzip2"
 	"compress/gzip"
 	"fmt"
 	"github.com/mkrautz/goar"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/go-xz"
+	"github.com/smira/lzma"
 	"io"
 	"os"
 	"strings"
@@ -24,16 +26,16 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	for {
 		header, err := library.Next()
 		if err == io.EOF {
-			return nil, fmt.Errorf("unable to find control.tar.gz part")
+			return nil, fmt.Errorf("unable to find control.tar.gz part in package %s", packageFile)
 		}
 		if err != nil {
-			return nil, fmt.Errorf("unable to read .deb archive: %s", err)
+			return nil, fmt.Errorf("unable to read .deb archive %s: %s", packageFile, err)
 		}
 
 		if header.Name == "control.tar.gz" {
 			ungzip, err := gzip.NewReader(library)
 			if err != nil {
-				return nil, fmt.Errorf("unable to ungzip: %s", err)
+				return nil, fmt.Errorf("unable to ungzip control file from %s. Error: %s", packageFile, err)
 			}
 			defer ungzip.Close()
 
@@ -41,15 +43,15 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 			for {
 				tarHeader, err := untar.Next()
 				if err == io.EOF {
-					return nil, fmt.Errorf("unable to find control file")
+					return nil, fmt.Errorf("unable to find control file in %s", packageFile)
 				}
 				if err != nil {
-					return nil, fmt.Errorf("unable to read .tar archive: %s", err)
+					return nil, fmt.Errorf("unable to read .tar archive from %s. Error: %s", packageFile, err)
 				}
 
-				if tarHeader.Name == "./control" {
+				if tarHeader.Name == "./control" || tarHeader.Name == "control" {
 					reader := NewControlFileReader(untar)
-					stanza, err := reader.ReadStanza()
+					stanza, err := reader.ReadStanza(false)
 					if err != nil {
 						return nil, err
 					}
@@ -69,16 +71,16 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 	}
 	defer file.Close()
 
-	line, err := bufio.NewReader(file).ReadString('\n')
+	isClearSigned, err := verifier.IsClearSigned(file)
+	file.Seek(0, 0)
+
 	if err != nil {
 		return nil, err
 	}
 
-	file.Seek(0, 0)
-
 	var text *os.File
 
-	if strings.Index(line, "BEGIN PGP SIGN") != -1 {
+	if isClearSigned {
 		text, err = verifier.ExtractClearsigned(file)
 		if err != nil {
 			return nil, err
@@ -89,7 +91,7 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 	}
 
 	reader := NewControlFileReader(text)
-	stanza, err := reader.ReadStanza()
+	stanza, err := reader.ReadStanza(false)
 	if err != nil {
 		return nil, err
 	}
@@ -97,3 +99,75 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 	return stanza, nil
 
 }
+
+// GetContentsFromDeb returns list of files installed by .deb package
+func GetContentsFromDeb(packageFile string) ([]string, error) {
+	file, err := os.Open(packageFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	library := ar.NewReader(file)
+	for {
+		header, err := library.Next()
+		if err == io.EOF {
+			return nil, fmt.Errorf("unable to find data.tar.* part in %s", packageFile)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("unable to read .deb archive from %s: %s", packageFile, err)
+		}
+
+		if strings.HasPrefix(header.Name, "data.tar") {
+			var tarInput io.Reader
+
+			switch header.Name {
+			case "data.tar":
+				tarInput = library
+			case "data.tar.gz":
+				ungzip, err := gzip.NewReader(library)
+				if err != nil {
+					return nil, fmt.Errorf("unable to ungzip data.tar.gz from %s: %s", packageFile, err)
+				}
+				defer ungzip.Close()
+				tarInput = ungzip
+			case "data.tar.bz2":
+				tarInput = bzip2.NewReader(library)
+			case "data.tar.xz":
+				unxz, err := xz.NewReader(library)
+				if err != nil {
+					return nil, fmt.Errorf("unable to unxz data.tar.xz from %s: %s", packageFile, err)
+				}
+				defer unxz.Close()
+				tarInput = unxz
+			case "data.tar.lzma":
+				unlzma := lzma.NewReader(library)
+				defer unlzma.Close()
+				tarInput = unlzma
+			default:
+				return nil, fmt.Errorf("unsupported tar compression in %s: %s", packageFile, header.Name)
+			}
+
+			untar := tar.NewReader(tarInput)
+			var results []string
+			for {
+				tarHeader, err := untar.Next()
+				if err == io.EOF {
+					return results, nil
+				}
+				if err != nil {
+					return nil, fmt.Errorf("unable to read .tar archive from %s: %s", packageFile, err)
+				}
+
+				if tarHeader.Typeflag == tar.TypeDir {
+					continue
+				}
+
+				if strings.HasPrefix(tarHeader.Name, "./") {
+					tarHeader.Name = tarHeader.Name[2:]
+				}
+				results = append(results, tarHeader.Name)
+			}
+		}
+	}
+}

deb/deb_test.go

@@ -2,13 +2,14 @@ package deb
 
 import (
 	"github.com/smira/aptly/utils"
-	. "launchpad.net/gocheck"
 	"path/filepath"
 	"runtime"
+
+	. "gopkg.in/check.v1"
 )
 
 type DebSuite struct {
-	debFile, dscFile, dscFileNoSign string
+	debFile, debFile2, dscFile, dscFileNoSign string
 }
 
 var _ = Suite(&DebSuite{})
@@ -16,6 +17,7 @@ var _ = Suite(&DebSuite{})
 func (s *DebSuite) SetUpSuite(c *C) {
 	_, _File, _, _ := runtime.Caller(0)
 	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	s.debFile2 = filepath.Join(filepath.Dir(_File), "../system/changes/hardlink_0.2.1_amd64.deb")
 	s.dscFile = filepath.Join(filepath.Dir(_File), "../system/files/pyspi_0.6.1-1.3.dsc")
 	s.dscFileNoSign = filepath.Join(filepath.Dir(_File), "../system/files/pyspi-0.6.1-1.3.stripped.dsc")
 }
@@ -26,7 +28,7 @@ func (s *DebSuite) TestGetControlFileFromDeb(c *C) {
 
 	_, _File, _, _ := runtime.Caller(0)
 	_, err = GetControlFileFromDeb(_File)
-	c.Check(err, ErrorMatches, "unable to read .deb archive: ar: missing global header")
+	c.Check(err, ErrorMatches, "^.+ar: missing global header")
 
 	st, err := GetControlFileFromDeb(s.debFile)
 	c.Check(err, IsNil)
@@ -54,3 +56,15 @@ func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
 	c.Check(st["Version"], Equals, "0.6.1-1.4")
 	c.Check(st["Source"], Equals, "pyspi")
 }
+
+func (s *DebSuite) TestGetContentsFromDeb(c *C) {
+	contents, err := GetContentsFromDeb(s.debFile)
+	c.Check(err, IsNil)
+	c.Check(contents, DeepEquals, []string{"usr/share/doc/libboost-program-options-dev/changelog.gz",
+		"usr/share/doc/libboost-program-options-dev/copyright"})
+
+	contents, err = GetContentsFromDeb(s.debFile2)
+	c.Check(err, IsNil)
+	c.Check(contents, DeepEquals, []string{"usr/bin/hardlink", "usr/share/man/man1/hardlink.1.gz",
+		"usr/share/doc/hardlink/changelog.gz", "usr/share/doc/hardlink/copyright", "usr/share/doc/hardlink/NEWS.Debian.gz"})
+}

deb/debian_test.go

@@ -1,8 +1,9 @@
 package deb
 
 import (
-	. "launchpad.net/gocheck"
 	"testing"
+
+	. "gopkg.in/check.v1"
 )
 
 // Launch gocheck tests

deb/format.go

@@ -5,15 +5,85 @@ import (
 	"errors"
 	"io"
 	"strings"
+	"unicode"
 )
 
 // Stanza or paragraph of Debian control file
 type Stanza map[string]string
 
 // Canonical order of fields in stanza
-var canocialOrder = []string{"Package", "Origin", "Label", "Suite", "Version", "Installed-Size", "Priority", "Section", "Maintainer",
+// Taken from: http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/vivid/apt/vivid/view/head:/apt-pkg/tagfile.cc#L504
-	"Architecture", "Codename", "Date", "Architectures", "Components", "Description", "MD5sum", "MD5Sum", "SHA1", "SHA256",
+var (
-	"Archive", "Component"}
+	canonicalOrderRelease = []string{
+		"Origin",
+		"Label",
+		"Archive",
+		"Suite",
+		"Version",
+		"Codename",
+		"Date",
+		"Architectures",
+		"Architecture",
+		"Components",
+		"Component",
+		"Description",
+		"MD5Sum",
+		"SHA1",
+		"SHA256",
+		"SHA512",
+	}
+
+	canonicalOrderBinary = []string{
+		"Package",
+		"Essential",
+		"Status",
+		"Priority",
+		"Section",
+		"Installed-Size",
+		"Maintainer",
+		"Original-Maintainer",
+		"Architecture",
+		"Source",
+		"Version",
+		"Replaces",
+		"Provides",
+		"Depends",
+		"Pre-Depends",
+		"Recommends",
+		"Suggests",
+		"Conflicts",
+		"Breaks",
+		"Conffiles",
+		"Filename",
+		"Size",
+		"MD5Sum",
+		"MD5sum",
+		"SHA1",
+		"SHA256",
+		"SHA512",
+		"Description",
+	}
+
+	canonicalOrderSource = []string{
+		"Package",
+		"Source",
+		"Binary",
+		"Version",
+		"Priority",
+		"Section",
+		"Maintainer",
+		"Original-Maintainer",
+		"Build-Depends",
+		"Build-Depends-Indep",
+		"Build-Conflicts",
+		"Build-Conflicts-Indep",
+		"Architecture",
+		"Standards-Version",
+		"Format",
+		"Directory",
+		"Files",
+	}
+)
 
 // Copy returns copy of Stanza
 func (s Stanza) Copy() (result Stanza) {
@@ -24,16 +94,46 @@ func (s Stanza) Copy() (result Stanza) {
 	return
 }
 
-// Write single field from Stanza to writer
+func isMultilineField(field string, isRelease bool) bool {
-func writeField(w *bufio.Writer, field, value string) (err error) {
+	switch field {
-	_, multiline := multilineFields[field]
+	case "Description":
+		return true
+	case "Files":
+		return true
+	case "Changes":
+		return true
+	case "Checksums-Sha1":
+		return true
+	case "Checksums-Sha256":
+		return true
+	case "Checksums-Sha512":
+		return true
+	case "Package-List":
+		return true
+	case "MD5Sum":
+		return isRelease
+	case "SHA1":
+		return isRelease
+	case "SHA256":
+		return isRelease
+	case "SHA512":
+		return isRelease
+	}
+	return false
+}
 
-	if !multiline {
+// Write single field from Stanza to writer
+func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) {
+	if !isMultilineField(field, isRelease) {
 		_, err = w.WriteString(field + ": " + value + "\n")
 	} else {
 		if !strings.HasSuffix(value, "\n") {
 			value = value + "\n"
 		}
+
+		if field != "Description" {
+			value = "\n" + value
+		}
 		_, err = w.WriteString(field + ":" + value)
 	}
 
@@ -41,12 +141,20 @@ func writeField(w *bufio.Writer, field, value string) (err error) {
 }
 
 // WriteTo saves stanza back to stream, modifying itself on the fly
-func (s Stanza) WriteTo(w *bufio.Writer) error {
+func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
-	for _, field := range canocialOrder {
+	canonicalOrder := canonicalOrderBinary
+	if isSource {
+		canonicalOrder = canonicalOrderSource
+	}
+	if isRelease {
+		canonicalOrder = canonicalOrderRelease
+	}
+
+	for _, field := range canonicalOrder {
 		value, ok := s[field]
 		if ok {
 			delete(s, field)
-			err := writeField(w, field, value)
+			err := writeField(w, field, value, isRelease)
 			if err != nil {
 				return err
 			}
@@ -54,7 +162,7 @@ func (s Stanza) WriteTo(w *bufio.Writer) error {
 	}
 
 	for field, value := range s {
-		err := writeField(w, field, value)
+		err := writeField(w, field, value, isRelease)
 		if err != nil {
 			return err
 		}
@@ -68,18 +176,33 @@ var (
 	ErrMalformedStanza = errors.New("malformed stanza syntax")
 )
 
-var multilineFields = make(map[string]bool)
+func canonicalCase(field string) string {
+	upper := strings.ToUpper(field)
+	switch upper {
+	case "SHA1", "SHA256", "SHA512":
+		return upper
+	case "MD5SUM":
+		return "MD5Sum"
+	case "NOTAUTOMATIC":
+		return "NotAutomatic"
+	case "BUTAUTOMATICUPGRADES":
+		return "ButAutomaticUpgrades"
+	}
 
-func init() {
+	startOfWord := true
-	multilineFields["Description"] = true
+
-	multilineFields["Files"] = true
+	return strings.Map(func(r rune) rune {
-	multilineFields["Changes"] = true
+		if startOfWord {
-	multilineFields["Checksums-Sha1"] = true
+			startOfWord = false
-	multilineFields["Checksums-Sha256"] = true
+			return unicode.ToUpper(r)
-	multilineFields["Package-List"] = true
+		}
-	multilineFields["SHA256"] = true
+
-	multilineFields["SHA1"] = true
+		if r == '-' {
-	multilineFields["MD5Sum"] = true
+			startOfWord = true
+		}
+
+		return unicode.ToLower(r)
+	}, field)
 }
 
 // ControlFileReader implements reading of control files stanza by stanza
@@ -93,7 +216,7 @@ func NewControlFileReader(r io.Reader) *ControlFileReader {
 }
 
 // ReadStanza reeads one stanza from control file
-func (c *ControlFileReader) ReadStanza() (Stanza, error) {
+func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
 	stanza := make(Stanza, 32)
 	lastField := ""
 	lastFieldMultiline := false
@@ -113,15 +236,15 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 			if lastFieldMultiline {
 				stanza[lastField] += line + "\n"
 			} else {
-				stanza[lastField] += strings.TrimSpace(line)
+				stanza[lastField] += " " + strings.TrimSpace(line)
 			}
 		} else {
 			parts := strings.SplitN(line, ":", 2)
 			if len(parts) != 2 {
 				return nil, ErrMalformedStanza
 			}
-			lastField = parts[0]
+			lastField = canonicalCase(parts[0])
-			_, lastFieldMultiline = multilineFields[lastField]
+			lastFieldMultiline = isMultilineField(lastField, isRelease)
 			if lastFieldMultiline {
 				stanza[lastField] = parts[1]
 				if parts[1] != "" {

deb/format_test.go

@@ -3,8 +3,9 @@ package deb
 import (
 	"bufio"
 	"bytes"
-	. "launchpad.net/gocheck"
 	"strings"
+
+	. "gopkg.in/check.v1"
 )
 
 type ControlFileSuite struct {
@@ -83,18 +84,18 @@ func (s *ControlFileSuite) SetUpTest(c *C) {
 func (s *ControlFileSuite) TestReadStanza(c *C) {
 	r := NewControlFileReader(s.reader)
 
-	stanza1, err := r.ReadStanza()
+	stanza1, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
-	stanza2, err := r.ReadStanza()
+	stanza2, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
-	stanza3, err := r.ReadStanza()
+	stanza3, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 	c.Assert(stanza3, IsNil)
 
 	c.Check(stanza1["Format"], Equals, "3.0 (quilt)")
-	c.Check(stanza1["Build-Depends"], Equals, "debhelper (>= 8),bash-completion (>= 1:1.1-3),libcurl4-nss-dev, libreadline-dev, libxml2-dev, libpcre3-dev, liboauth-dev, xsltproc, docbook-xsl, docbook-xml, dh-autoreconf")
+	c.Check(stanza1["Build-Depends"], Equals, "debhelper (>= 8), bash-completion (>= 1:1.1-3), libcurl4-nss-dev, libreadline-dev, libxml2-dev, libpcre3-dev, liboauth-dev, xsltproc, docbook-xsl, docbook-xml, dh-autoreconf")
 	c.Check(stanza1["Files"], Equals, " 3d5f65778bf3f89be03c313b0024b62c 1980 bti_032-1.dsc\n"+
 		" 1e0d0b693fdeebec268004ba41701baf 59773 bti_032.orig.tar.gz\n"+" ac1229a6d685023aeb8fcb0806324aa8 5065 bti_032-1.debian.tar.gz\n")
 	c.Check(len(stanza2), Equals, 20)
@@ -102,12 +103,12 @@ func (s *ControlFileSuite) TestReadStanza(c *C) {
 
 func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 	r := NewControlFileReader(s.reader)
-	stanza, err := r.ReadStanza()
+	stanza, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
 	buf := &bytes.Buffer{}
 	w := bufio.NewWriter(buf)
-	err = stanza.Copy().WriteTo(w)
+	err = stanza.Copy().WriteTo(w, true, false)
 	c.Assert(err, IsNil)
 	err = w.Flush()
 	c.Assert(err, IsNil)
@@ -115,19 +116,31 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 	str := buf.String()
 
 	r = NewControlFileReader(buf)
-	stanza2, err := r.ReadStanza()
+	stanza2, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
 	c.Assert(stanza2, DeepEquals, stanza)
 	c.Assert(strings.HasPrefix(str, "Package: "), Equals, true)
 }
 
+func (s *ControlFileSuite) TestCanonicalCase(c *C) {
+	c.Check(canonicalCase("Package"), Equals, "Package")
+	c.Check(canonicalCase("package"), Equals, "Package")
+	c.Check(canonicalCase("pAckaGe"), Equals, "Package")
+	c.Check(canonicalCase("MD5Sum"), Equals, "MD5Sum")
+	c.Check(canonicalCase("SHA1"), Equals, "SHA1")
+	c.Check(canonicalCase("SHA256"), Equals, "SHA256")
+	c.Check(canonicalCase("Package-List"), Equals, "Package-List")
+	c.Check(canonicalCase("package-list"), Equals, "Package-List")
+	c.Check(canonicalCase("packaGe-lIst"), Equals, "Package-List")
+}
+
 func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
 	for i := 0; i < c.N; i++ {
 		reader := bytes.NewBufferString(controlFile)
 		r := NewControlFileReader(reader)
 		for {
-			s, e := r.ReadStanza()
+			s, e := r.ReadStanza(false)
 			if s == nil && e == nil {
 				break
 			}

deb/graph.go

@@ -0,0 +1,120 @@
+package deb
+
+import (
+	"fmt"
+	"github.com/awalterschulze/gographviz"
+	"strings"
+)
+
+// BuildGraph generates graph contents from aptly object database
+func BuildGraph(collectionFactory *CollectionFactory) (gographviz.Interface, error) {
+	var err error
+
+	graph := gographviz.NewEscape()
+	graph.SetDir(true)
+	graph.SetName("aptly")
+
+	existingNodes := map[string]bool{}
+
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *RemoteRepo) error {
+		err := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+
+		graph.AddNode("aptly", repo.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "darkgoldenrod1",
+			"label": fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
+				repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "),
+				strings.Join(repo.Architectures, ", "), repo.NumPackages()),
+		})
+		existingNodes[repo.UUID] = true
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *LocalRepo) error {
+		err := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+
+		graph.AddNode("aptly", repo.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "mediumseagreen",
+			"label": fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
+				repo.Name, repo.Comment, repo.NumPackages()),
+		})
+		existingNodes[repo.UUID] = true
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
+		existingNodes[snapshot.UUID] = true
+		return nil
+	})
+
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
+		err := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+
+		description := snapshot.Description
+		if snapshot.SourceKind == "repo" {
+			description = "Snapshot from repo"
+		}
+
+		graph.AddNode("aptly", snapshot.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "cadetblue1",
+			"label":     fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages()),
+		})
+
+		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
+			for _, uuid := range snapshot.SourceIDs {
+				_, exists := existingNodes[uuid]
+				if exists {
+					graph.AddEdge(uuid, snapshot.UUID, true, nil)
+				}
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	collectionFactory.PublishedRepoCollection().ForEach(func(repo *PublishedRepo) error {
+		graph.AddNode("aptly", repo.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "darkolivegreen1",
+			"label": fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution,
+				strings.Join(repo.Components(), " "), strings.Join(repo.Architectures, ", ")),
+		})
+
+		for _, uuid := range repo.Sources {
+			_, exists := existingNodes[uuid]
+			if exists {
+				graph.AddEdge(uuid, repo.UUID, true, nil)
+			}
+		}
+
+		return nil
+	})
+
+	return graph, nil
+}

deb/import.go

@@ -0,0 +1,193 @@
+package deb
+
+import (
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// CollectPackageFiles walks filesystem collecting all candidates for package files
+func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, failedFiles []string) {
+	for _, location := range locations {
+		info, err2 := os.Stat(location)
+		if err2 != nil {
+			reporter.Warning("Unable to process %s: %s", location, err2)
+			failedFiles = append(failedFiles, location)
+			continue
+		}
+		if info.IsDir() {
+			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
+				if err3 != nil {
+					return err3
+				}
+				if info.IsDir() {
+					return nil
+				}
+
+				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+					strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
+					packageFiles = append(packageFiles, path)
+				}
+
+				return nil
+			})
+
+			if err2 != nil {
+				reporter.Warning("Unable to process %s: %s", location, err2)
+				failedFiles = append(failedFiles, location)
+				continue
+			}
+		} else {
+			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+				strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
+				packageFiles = append(packageFiles, location)
+			} else {
+				reporter.Warning("Unknown file extension: %s", location)
+				failedFiles = append(failedFiles, location)
+				continue
+			}
+		}
+	}
+
+	sort.Strings(packageFiles)
+
+	return
+}
+
+// ImportPackageFiles imports files into local repository
+func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier utils.Verifier,
+	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter, restriction PackageQuery) (processedFiles []string, failedFiles []string, err error) {
+	if forceReplace {
+		list.PrepareIndex()
+	}
+
+	for _, file := range packageFiles {
+		var (
+			stanza Stanza
+			p      *Package
+		)
+
+		candidateProcessedFiles := []string{}
+		isSourcePackage := strings.HasSuffix(file, ".dsc")
+		isUdebPackage := strings.HasSuffix(file, ".udeb")
+
+		if isSourcePackage {
+			stanza, err = GetControlFileFromDsc(file, verifier)
+
+			if err == nil {
+				stanza["Package"] = stanza["Source"]
+				delete(stanza, "Source")
+
+				p, err = NewSourcePackageFromControlFile(stanza)
+			}
+		} else {
+			stanza, err = GetControlFileFromDeb(file)
+			if isUdebPackage {
+				p = NewUdebPackageFromControlFile(stanza)
+			} else {
+				p = NewPackageFromControlFile(stanza)
+			}
+		}
+		if err != nil {
+			reporter.Warning("Unable to read file %s: %s", file, err)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		if p.Name == "" {
+			reporter.Warning("Empty package name on %s", file)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		if p.Version == "" {
+			reporter.Warning("Empty version on %s", file)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		if p.Architecture == "" {
+			reporter.Warning("Empty architecture on %s", file)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		var checksums utils.ChecksumInfo
+		checksums, err = utils.ChecksumsForFile(file)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if isSourcePackage {
+			p.UpdateFiles(append(p.Files(), PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
+		} else {
+			p.UpdateFiles([]PackageFile{{Filename: filepath.Base(file), Checksums: checksums}})
+		}
+
+		err = pool.Import(file, checksums.MD5)
+		if err != nil {
+			reporter.Warning("Unable to import file %s into pool: %s", file, err)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		candidateProcessedFiles = append(candidateProcessedFiles, file)
+
+		// go over all files, except for the last one (.dsc/.deb itself)
+		for _, f := range p.Files() {
+			if filepath.Base(f.Filename) == filepath.Base(file) {
+				continue
+			}
+			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
+			err = pool.Import(sourceFile, f.Checksums.MD5)
+			if err != nil {
+				reporter.Warning("Unable to import file %s into pool: %s", sourceFile, err)
+				failedFiles = append(failedFiles, file)
+				break
+			}
+
+			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
+		}
+		if err != nil {
+			// some files haven't been imported
+			continue
+		}
+
+		if restriction != nil && !restriction.Matches(p) {
+			reporter.Warning("%s has been ignored as it doesn't match restriction", p)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		err = collection.Update(p)
+		if err != nil {
+			reporter.Warning("Unable to save package %s: %s", p, err)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		if forceReplace {
+			conflictingPackages := list.Search(Dependency{Pkg: p.Name, Version: p.Version, Relation: VersionEqual, Architecture: p.Architecture}, true)
+			for _, cp := range conflictingPackages {
+				reporter.Removed("%s removed due to conflict with package being added", cp)
+				list.Remove(cp)
+			}
+		}
+
+		err = list.Add(p)
+		if err != nil {
+			reporter.Warning("Unable to add package to repo %s: %s", p, err)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
+		reporter.Added("%s added", p)
+		processedFiles = append(processedFiles, candidateProcessedFiles...)
+	}
+
+	err = nil
+	return
+}

deb/index_files.go

@@ -24,6 +24,7 @@ type indexFile struct {
 	parent       *indexFiles
 	discardable  bool
 	compressable bool
+	onlyGzip     bool
 	signable     bool
 	relativePath string
 	tempFilename string
@@ -73,6 +74,9 @@ func (file *indexFile) Finalize(signer utils.Signer) error {
 	exts := []string{""}
 	if file.compressable {
 		exts = append(exts, ".gz", ".bz2")
+		if file.onlyGzip {
+			exts = []string{".gz"}
+		}
 	}
 
 	for _, ext := range exts {
@@ -150,7 +154,10 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 }
 
 func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
-	key := fmt.Sprintf("pi-%s-%s-%s", component, arch, udeb)
+	if arch == "source" {
+		udeb = false
+	}
+	key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb)
 	file, ok := files.indexes[key]
 	if !ok {
 		var relativePath string
@@ -180,7 +187,10 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 }
 
 func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexFile {
-	key := fmt.Sprintf("ri-%s-%s-%s", component, arch, udeb)
+	if arch == "source" {
+		udeb = false
+	}
+	key := fmt.Sprintf("ri-%s-%s-%v", component, arch, udeb)
 	file, ok := files.indexes[key]
 	if !ok {
 		var relativePath string
@@ -209,6 +219,36 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 	return file
 }
 
+func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *indexFile {
+	if arch == "source" {
+		udeb = false
+	}
+	key := fmt.Sprintf("ci-%s-%s-%v", component, arch, udeb)
+	file, ok := files.indexes[key]
+	if !ok {
+		var relativePath string
+
+		if udeb {
+			relativePath = filepath.Join(component, fmt.Sprintf("Contents-udeb-%s", arch))
+		} else {
+			relativePath = filepath.Join(component, fmt.Sprintf("Contents-%s", arch))
+		}
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  true,
+			compressable: true,
+			onlyGzip:     true,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
 func (files *indexFiles) ReleaseFile() *indexFile {
 	return &indexFile{
 		parent:       files,

deb/list.go

@@ -36,6 +36,15 @@ type PackageList struct {
 	packagesIndex []*Package
 	// Map of packages for each virtual package (provides)
 	providesIndex map[string][]*Package
+	// Package key generation function
+	keyFunc func(p *Package) string
+	// Allow duplicates?
+	duplicatesAllowed bool
+}
+
+// PackageConflictError means that package can't be added to the list due to error
+type PackageConflictError struct {
+	error
 }
 
 // Verify interface
@@ -44,9 +53,35 @@ var (
 	_ PackageCatalog = &PackageList{}
 )
 
-// NewPackageList creates empty package list
+func packageShortKey(p *Package) string {
+	return string(p.ShortKey(""))
+}
+
+func packageFullKey(p *Package) string {
+	return string(p.Key(""))
+}
+
+// NewPackageList creates empty package list without duplicate package
 func NewPackageList() *PackageList {
-	return &PackageList{packages: make(map[string]*Package, 1000)}
+	return NewPackageListWithDuplicates(false, 1000)
+}
+
+func NewPackageListWithDuplicates(duplicates bool, capacity int) *PackageList {
+	if capacity == 0 {
+		capacity = 1000
+	}
+
+	result := &PackageList{
+		packages:          make(map[string]*Package, capacity),
+		duplicatesAllowed: duplicates,
+		keyFunc:           packageShortKey,
+	}
+
+	if duplicates {
+		result.keyFunc = packageFullKey
+	}
+
+	return result
 }
 
 // NewPackageListFromRefList loads packages list from PackageRefList
@@ -56,7 +91,7 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 		return NewPackageList(), nil
 	}
 
-	result := &PackageList{packages: make(map[string]*Package, reflist.Len())}
+	result := NewPackageListWithDuplicates(false, reflist.Len())
 
 	if progress != nil {
 		progress.InitBar(int64(reflist.Len()), false)
@@ -86,11 +121,11 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 
 // Add appends package to package list, additionally checking for uniqueness
 func (l *PackageList) Add(p *Package) error {
-	key := string(p.ShortKey(""))
+	key := l.keyFunc(p)
 	existing, ok := l.packages[key]
 	if ok {
 		if !existing.Equals(p) {
-			return fmt.Errorf("conflict in package %s", p)
+			return &PackageConflictError{fmt.Errorf("conflict in package %s", p)}
 		}
 		return nil
 	}
@@ -165,7 +200,7 @@ func (l *PackageList) Append(pl *PackageList) error {
 
 // Remove removes package from the list, and updates index when required
 func (l *PackageList) Remove(p *Package) {
-	delete(l.packages, string(p.ShortKey("")))
+	delete(l.packages, l.keyFunc(p))
 	if l.indexed {
 		for _, provides := range p.Provides {
 			for i, pkg := range l.providesIndex[provides] {
@@ -205,6 +240,19 @@ func (l *PackageList) Architectures(includeSource bool) (result []string) {
 	return
 }
 
+// Strings builds list of strings with package keys
+func (l *PackageList) Strings() []string {
+	result := make([]string, l.Len())
+	i := 0
+
+	for _, p := range l.packages {
+		result[i] = string(p.Key(""))
+		i += 1
+	}
+
+	return result
+}
+
 // depSliceDeduplicate removes dups in slice of Dependencies
 func depSliceDeduplicate(s []Dependency) []Dependency {
 	l := len(s)
@@ -234,7 +282,7 @@ func depSliceDeduplicate(s []Dependency) []Dependency {
 
 // VerifyDependencies looks for missing dependencies in package list.
 //
-// Analysis would be peformed for each architecture, in specified sources
+// Analysis would be performed for each architecture, in specified sources
 func (l *PackageList) VerifyDependencies(options int, architectures []string, sources *PackageList, progress aptly.Progress) ([]Dependency, error) {
 	l.PrepareIndex()
 	missing := make([]Dependency, 0, 128)
@@ -347,7 +395,7 @@ func (l *PackageList) PrepareIndex() {
 
 // Scan searches package index using full scan
 func (l *PackageList) Scan(q PackageQuery) (result *PackageList) {
-	result = NewPackageList()
+	result = NewPackageListWithDuplicates(l.duplicatesAllowed, 0)
 	for _, pkg := range l.packages {
 		if q.Matches(pkg) {
 			result.Add(pkg)
@@ -364,7 +412,7 @@ func (l *PackageList) SearchSupported() bool {
 
 // SearchByKey looks up package by exact key reference
 func (l *PackageList) SearchByKey(arch, name, version string) (result *PackageList) {
-	result = NewPackageList()
+	result = NewPackageListWithDuplicates(l.duplicatesAllowed, 0)
 
 	pkg := l.packages["P"+arch+" "+name+" "+version]
 	if pkg != nil {

deb/list_test.go

@@ -2,10 +2,11 @@ package deb
 
 import (
 	"errors"
-	. "launchpad.net/gocheck"
 	"regexp"
 	"sort"
 	"strings"
+
+	. "gopkg.in/check.v1"
 )
 
 type containsChecker struct {
@@ -79,20 +80,20 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 
 	s.il = NewPackageList()
 	s.packages = []*Package{
-		&Package{Name: "lib", Version: "1.0", Architecture: "i386", Source: "lib (0.9)", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"mail-agent"}}},
+		{Name: "lib", Version: "1.0", Architecture: "i386", Source: "lib (0.9)", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"mail-agent"}}},
-		&Package{Name: "dpkg", Version: "1.7", Architecture: "i386", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
+		{Name: "dpkg", Version: "1.7", Architecture: "i386", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
-		&Package{Name: "data", Version: "1.1~bp1", Architecture: "all", Source: "app", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
+		{Name: "data", Version: "1.1~bp1", Architecture: "all", Source: "app", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "1.1~bp1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
-		&Package{Name: "mailer", Version: "3.5.8", Architecture: "i386", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		{Name: "mailer", Version: "3.5.8", Architecture: "i386", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "1.1~bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
+		{Name: "app", Version: "1.1~bp1", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
-		&Package{Name: "app", Version: "1.0", Architecture: "s390", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "1.0", Architecture: "s390", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
-		&Package{Name: "aa", Version: "2.0-1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
+		{Name: "aa", Version: "2.0-1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
-		&Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "amd64", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
+		{Name: "dpkg", Version: "1.6.1-3", Architecture: "amd64", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
-		&Package{Name: "libx", Version: "1.5", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
+		{Name: "libx", Version: "1.5", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
-		&Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "arm", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
+		{Name: "dpkg", Version: "1.6.1-3", Architecture: "arm", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
-		&Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "dpkg", Version: "1.6.1-3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
-		&Package{Name: "dpkg", Version: "1.7", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "dpkg", Version: "1.7", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
 	}
 	for _, p := range s.packages {
 		s.il.Add(p)
@@ -101,12 +102,12 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 
 	s.il2 = NewPackageList()
 	s.packages2 = []*Package{
-		&Package{Name: "mailer", Version: "3.5.8", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		{Name: "mailer", Version: "3.5.8", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
-		&Package{Name: "sendmail", Version: "1.0", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		{Name: "sendmail", Version: "1.0", Architecture: "amd64", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
-		&Package{Name: "app", Version: "1.1-bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "1.1-bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
-		&Package{Name: "app", Version: "1.1-bp2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "1.1-bp2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
-		&Package{Name: "app", Version: "1.2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
+		{Name: "app", Version: "1.2", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
-		&Package{Name: "app", Version: "3.0", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		{Name: "app", Version: "3.0", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
 	}
 	for _, p := range s.packages2 {
 		s.il2.Add(p)
@@ -114,10 +115,10 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 	s.il2.PrepareIndex()
 
 	s.sourcePackages = []*Package{
-		&Package{Name: "postfix", Version: "1.3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "postfix", Version: "1.3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "app", Version: "1.1~bp1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
-		&Package{Name: "aa", Version: "2.0-1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "aa", Version: "2.0-1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
-		&Package{Name: "lib", Version: "0.9", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		{Name: "lib", Version: "0.9", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
 	}
 
 }
@@ -378,6 +379,20 @@ func (s *PackageListSuite) TestFilter(c *C) {
 		&FieldQuery{Field: "$Architecture", Relation: VersionRegexp, Value: "i.*6", Regexp: regexp.MustCompile("i.*6")}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")},
+		&NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}},
+	}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}},
+		&FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")},
+	}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386")
 }
 
 func (s *PackageListSuite) TestVerifyDependencies(c *C) {
@@ -387,7 +402,7 @@ func (s *PackageListSuite) TestVerifyDependencies(c *C) {
 
 	missing, err = s.il.VerifyDependencies(0, []string{"i386", "amd64"}, s.il, nil)
 	c.Check(err, IsNil)
-	c.Check(missing, DeepEquals, []Dependency{Dependency{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "amd64"}})
+	c.Check(missing, DeepEquals, []Dependency{{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "amd64"}})
 
 	missing, err = s.il.VerifyDependencies(0, []string{"arm"}, s.il, nil)
 	c.Check(err, IsNil)
@@ -395,8 +410,8 @@ func (s *PackageListSuite) TestVerifyDependencies(c *C) {
 
 	missing, err = s.il.VerifyDependencies(DepFollowAllVariants, []string{"arm"}, s.il, nil)
 	c.Check(err, IsNil)
-	c.Check(missing, DeepEquals, []Dependency{Dependency{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "arm"},
+	c.Check(missing, DeepEquals, []Dependency{{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "arm"},
-		Dependency{Pkg: "mail-agent", Relation: VersionDontCare, Version: "", Architecture: "arm"}})
+		{Pkg: "mail-agent", Relation: VersionDontCare, Version: "", Architecture: "arm"}})
 
 	for _, p := range s.sourcePackages {
 		s.il.Add(p)
@@ -404,11 +419,11 @@ func (s *PackageListSuite) TestVerifyDependencies(c *C) {
 
 	missing, err = s.il.VerifyDependencies(DepFollowSource, []string{"i386", "amd64"}, s.il, nil)
 	c.Check(err, IsNil)
-	c.Check(missing, DeepEquals, []Dependency{Dependency{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "amd64"}})
+	c.Check(missing, DeepEquals, []Dependency{{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "amd64"}})
 
 	missing, err = s.il.VerifyDependencies(DepFollowSource, []string{"arm"}, s.il, nil)
 	c.Check(err, IsNil)
-	c.Check(missing, DeepEquals, []Dependency{Dependency{Pkg: "libx", Relation: VersionEqual, Version: "1.5", Architecture: "source"}})
+	c.Check(missing, DeepEquals, []Dependency{{Pkg: "libx", Relation: VersionEqual, Version: "1.5", Architecture: "source"}})
 
 	_, err = s.il.VerifyDependencies(0, []string{"i386", "amd64", "s390"}, s.il, nil)
 	c.Check(err, ErrorMatches, "unable to process package app_1.0_s390:.*")

deb/local.go

@@ -2,17 +2,18 @@ package deb
 
 import (
 	"bytes"
-	"code.google.com/p/go-uuid/uuid"
 	"fmt"
 	"github.com/smira/aptly/database"
+	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
 	"log"
+	"sync"
 )
 
 // LocalRepo is a collection of packages created locally
 type LocalRepo struct {
 	// Permanent internal ID
-	UUID string
+	UUID string `json:"-"`
 	// User-assigned name
 	Name string
 	// Comment
@@ -21,6 +22,8 @@ type LocalRepo struct {
 	DefaultDistribution string `codec:",omitempty"`
 	// DefaultComponent
 	DefaultComponent string `codec:",omitempty"`
+	// Uploaders configuration
+	Uploaders *Uploaders `code:",omitempty" json:"-"`
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 }
@@ -88,6 +91,7 @@ func (repo *LocalRepo) RefKey() []byte {
 
 // LocalRepoCollection does listing, updating/adding/deleting of LocalRepos
 type LocalRepoCollection struct {
+	*sync.RWMutex
 	db   database.Storage
 	list []*LocalRepo
 }
@@ -95,7 +99,8 @@ type LocalRepoCollection struct {
 // NewLocalRepoCollection loads LocalRepos from DB and makes up collection
 func NewLocalRepoCollection(db database.Storage) *LocalRepoCollection {
 	result := &LocalRepoCollection{
-		db: db,
+		RWMutex: &sync.RWMutex{},
+		db:      db,
 	}
 
 	blobs := db.FetchByPrefix([]byte("L"))
@@ -104,7 +109,7 @@ func NewLocalRepoCollection(db database.Storage) *LocalRepoCollection {
 	for _, blob := range blobs {
 		r := &LocalRepo{}
 		if err := r.Decode(blob); err != nil {
-			log.Printf("Error decoding mirror: %s\n", err)
+			log.Printf("Error decoding repo: %s\n", err)
 		} else {
 			result.list = append(result.list, r)
 		}

deb/local_test.go

@@ -3,7 +3,8 @@ package deb
 import (
 	"errors"
 	"github.com/smira/aptly/database"
-	. "launchpad.net/gocheck"
+
+	. "gopkg.in/check.v1"
 )
 
 type LocalRepoSuite struct {

deb/package.go

@@ -1,6 +1,7 @@
 package deb
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
@@ -31,13 +32,19 @@ type Package struct {
 	// Is this >= 0.6 package?
 	V06Plus bool
 	// Offload fields
-	deps  *PackageDependencies
+	deps     *PackageDependencies
-	extra *Stanza
+	extra    *Stanza
-	files *PackageFiles
+	files    *PackageFiles
+	contents []string
 	// Mother collection
 	collection *PackageCollection
 }
 
+// Check interface
+var (
+	_ json.Marshaler = &Package{}
+)
+
 // NewPackageFromControlFile creates Package from parsed Debian control file
 func NewPackageFromControlFile(input Stanza) *Package {
 	result := &Package{
@@ -55,21 +62,30 @@ func NewPackageFromControlFile(input Stanza) *Package {
 
 	filesize, _ := strconv.ParseInt(input["Size"], 10, 64)
 
+	md5, ok := input["MD5sum"]
+	if !ok {
+		// there are some broken repos out there with MD5 in wrong field
+		md5 = input["MD5Sum"]
+	}
+
 	result.UpdateFiles(PackageFiles{PackageFile{
 		Filename:     filepath.Base(input["Filename"]),
 		downloadPath: filepath.Dir(input["Filename"]),
 		Checksums: utils.ChecksumInfo{
 			Size:   filesize,
-			MD5:    strings.TrimSpace(input["MD5sum"]),
+			MD5:    strings.TrimSpace(md5),
 			SHA1:   strings.TrimSpace(input["SHA1"]),
 			SHA256: strings.TrimSpace(input["SHA256"]),
+			SHA512: strings.TrimSpace(input["SHA512"]),
 		},
 	}})
 
 	delete(input, "Filename")
 	delete(input, "MD5sum")
+	delete(input, "MD5Sum")
 	delete(input, "SHA1")
 	delete(input, "SHA256")
+	delete(input, "SHA512")
 	delete(input, "Size")
 
 	depends := &PackageDependencies{}
@@ -101,62 +117,20 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 	delete(input, "Version")
 	delete(input, "Architecture")
 
+	var err error
+
 	files := make(PackageFiles, 0, 3)
-
+	files, err = files.ParseSumFields(input)
-	parseSums := func(field string, setter func(sum *utils.ChecksumInfo, data string)) error {
-		for _, line := range strings.Split(input[field], "\n") {
-			line = strings.TrimSpace(line)
-			if line == "" {
-				continue
-			}
-			parts := strings.Fields(line)
-
-			if len(parts) != 3 {
-				return fmt.Errorf("unparseable hash sum line: %#v", line)
-			}
-
-			size, err := strconv.ParseInt(parts[1], 10, 64)
-			if err != nil {
-				return fmt.Errorf("unable to parse size: %s", err)
-			}
-
-			filename := filepath.Base(parts[2])
-
-			found := false
-			pos := 0
-			for i, file := range files {
-				if file.Filename == filename {
-					found = true
-					pos = i
-					break
-				}
-			}
-
-			if !found {
-				files = append(files, PackageFile{Filename: filename, downloadPath: input["Directory"]})
-				pos = len(files) - 1
-			}
-
-			files[pos].Checksums.Size = size
-			setter(&files[pos].Checksums, parts[0])
-		}
-
-		delete(input, field)
-
-		return nil
-	}
-
-	err := parseSums("Files", func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
 	if err != nil {
 		return nil, err
 	}
-	err = parseSums("Checksums-Sha1", func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
+
-	if err != nil {
+	delete(input, "Files")
-		return nil, err
+	delete(input, "Checksums-Sha1")
-	}
+	delete(input, "Checksums-Sha256")
-	err = parseSums("Checksums-Sha256", func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
+
-	if err != nil {
+	for i := range files {
-		return nil, err
+		files[i].downloadPath = input["Directory"]
 	}
 
 	result.UpdateFiles(files)
@@ -198,6 +172,21 @@ func (p *Package) String() string {
 	return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.Architecture)
 }
 
+// ExtendedStanza returns package stanza enhanced with aptly-specific fields
+func (p *Package) ExtendedStanza() Stanza {
+	stanza := p.Stanza()
+	stanza["FilesHash"] = fmt.Sprintf("%08x", p.FilesHash)
+	stanza["Key"] = string(p.Key(""))
+	stanza["ShortKey"] = string(p.ShortKey(""))
+
+	return stanza
+}
+
+// MarshalJSON implements json.Marshaller interface
+func (p *Package) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.ExtendedStanza())
+}
+
 // GetField returns fields from package
 func (p *Package) GetField(name string) string {
 	switch name {
@@ -262,7 +251,6 @@ func (p *Package) GetField(name string) string {
 	default:
 		return p.Extra()[name]
 	}
-	return ""
 }
 
 // MatchesArchitecture checks whether packages matches specified architecture
@@ -314,6 +302,21 @@ func (p *Package) MatchesDependency(dep Dependency) bool {
 	panic("unknown relation")
 }
 
+// GetName returns package name
+func (p *Package) GetName() string {
+	return p.Name
+}
+
+// GetVersion returns package version
+func (p *Package) GetVersion() string {
+	return p.Version
+}
+
+// GetArchitecture returns package arch
+func (p *Package) GetArchitecture() string {
+	return p.Architecture
+}
+
 // GetDependencies compiles list of dependenices by flags from options
 func (p *Package) GetDependencies(options int) (dependencies []string) {
 	deps := p.Deps()
@@ -350,6 +353,16 @@ func (p *Package) GetDependencies(options int) (dependencies []string) {
 	return
 }
 
+// QualifiedName returns [$SECTION/]$NAME
+func (p *Package) QualifiedName() string {
+	section := p.Extra()["Section"]
+	if section != "" {
+		return section + "/" + p.Name
+	}
+
+	return p.Name
+}
+
 // Extra returns Stanza of extra fields (it may load it from collection)
 func (p *Package) Extra() Stanza {
 	if p.extra == nil {
@@ -388,6 +401,35 @@ func (p *Package) Files() PackageFiles {
 	return *p.files
 }
 
+// Contents returns cached package contents
+func (p *Package) Contents(packagePool aptly.PackagePool) []string {
+	if p.IsSource {
+		return nil
+	}
+
+	return p.collection.loadContents(p, packagePool)
+}
+
+// CalculateContents looks up contents in package file
+func (p *Package) CalculateContents(packagePool aptly.PackagePool) []string {
+	if p.IsSource {
+		return nil
+	}
+
+	file := p.Files()[0]
+	path, err := packagePool.Path(file.Filename, file.Checksums.MD5)
+	if err != nil {
+		panic(err)
+	}
+
+	contents, err := GetContentsFromDeb(path)
+	if err != nil {
+		panic(err)
+	}
+
+	return contents
+}
+
 // UpdateFiles saves new state of files
 func (p *Package) UpdateFiles(files PackageFiles) {
 	p.files = &files
@@ -404,11 +446,13 @@ func (p *Package) Stanza() (result Stanza) {
 		result["Architecture"] = p.SourceArchitecture
 	} else {
 		result["Architecture"] = p.Architecture
-		result["Source"] = p.Source
+		if p.Source != "" {
+			result["Source"] = p.Source
+		}
 	}
 
 	if p.IsSource {
-		md5, sha1, sha256 := make([]string, 0), make([]string, 0), make([]string, 0)
+		md5, sha1, sha256, sha512 := []string{}, []string{}, []string{}, []string{}
 
 		for _, f := range p.Files() {
 			if f.Checksums.MD5 != "" {
@@ -420,11 +464,21 @@ func (p *Package) Stanza() (result Stanza) {
 			if f.Checksums.SHA256 != "" {
 				sha256 = append(sha256, fmt.Sprintf(" %s %d %s\n", f.Checksums.SHA256, f.Checksums.Size, f.Filename))
 			}
+			if f.Checksums.SHA512 != "" {
+				sha512 = append(sha512, fmt.Sprintf(" %s %d %s\n", f.Checksums.SHA512, f.Checksums.Size, f.Filename))
+			}
 		}
 
 		result["Files"] = strings.Join(md5, "")
-		result["Checksums-Sha1"] = strings.Join(sha1, "")
+		if len(sha1) > 0 {
-		result["Checksums-Sha256"] = strings.Join(sha256, "")
+			result["Checksums-Sha1"] = strings.Join(sha1, "")
+		}
+		if len(sha256) > 0 {
+			result["Checksums-Sha256"] = strings.Join(sha256, "")
+		}
+		if len(sha512) > 0 {
+			result["Checksums-Sha512"] = strings.Join(sha512, "")
+		}
 	} else {
 		f := p.Files()[0]
 		result["Filename"] = f.DownloadURL()
@@ -432,10 +486,13 @@ func (p *Package) Stanza() (result Stanza) {
 			result["MD5sum"] = f.Checksums.MD5
 		}
 		if f.Checksums.SHA1 != "" {
-			result["SHA1"] = " " + f.Checksums.SHA1
+			result["SHA1"] = f.Checksums.SHA1
 		}
 		if f.Checksums.SHA256 != "" {
-			result["SHA256"] = " " + f.Checksums.SHA256
+			result["SHA256"] = f.Checksums.SHA256
+		}
+		if f.Checksums.SHA512 != "" {
+			result["SHA512"] = f.Checksums.SHA512
 		}
 		result["Size"] = fmt.Sprintf("%d", f.Checksums.Size)
 	}

deb/package_collection.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"bytes"
 	"fmt"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/ugorji/go/codec"
 	"path/filepath"
@@ -10,9 +11,8 @@ import (
 
 // PackageCollection does management of packages in DB
 type PackageCollection struct {
-	db           database.Storage
+	db          database.Storage
-	encodeBuffer bytes.Buffer
+	codecHandle *codec.MsgpackHandle
-	codecHandle  *codec.MsgpackHandle
 }
 
 // Verify interface
@@ -161,45 +161,82 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 	return files
 }
 
+// loadContents loads or calculates and saves package contents
+func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.PackagePool) []string {
+	encoded, err := collection.db.Get(p.Key("xC"))
+	if err == nil {
+		contents := []string{}
+
+		decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
+		err = decoder.Decode(&contents)
+		if err != nil {
+			panic("unable to decode contents")
+		}
+
+		return contents
+	}
+
+	if err != database.ErrNotFound {
+		panic("unable to load contents")
+	}
+
+	contents := p.CalculateContents(packagePool)
+
+	var buf bytes.Buffer
+	err = codec.NewEncoder(&buf, collection.codecHandle).Encode(contents)
+	if err != nil {
+		panic("unable to encode contents")
+	}
+
+	err = collection.db.Put(p.Key("xC"), buf.Bytes())
+	if err != nil {
+		panic("unable to save contents")
+	}
+
+	return contents
+}
+
 // Update adds or updates information about package in DB checking for conficts first
 func (collection *PackageCollection) Update(p *Package) error {
-	encoder := codec.NewEncoder(&collection.encodeBuffer, collection.codecHandle)
+	var encodeBuffer bytes.Buffer
 
-	collection.encodeBuffer.Reset()
+	encoder := codec.NewEncoder(&encodeBuffer, collection.codecHandle)
-	collection.encodeBuffer.WriteByte(0xc1)
+
-	collection.encodeBuffer.WriteByte(0x1)
+	encodeBuffer.Reset()
+	encodeBuffer.WriteByte(0xc1)
+	encodeBuffer.WriteByte(0x1)
 	err := encoder.Encode(p)
 	if err != nil {
 		return err
 	}
 
-	err = collection.db.Put(p.Key(""), collection.encodeBuffer.Bytes())
+	err = collection.db.Put(p.Key(""), encodeBuffer.Bytes())
 	if err != nil {
 		return err
 	}
 
 	// Encode offloaded fields one by one
 	if p.files != nil {
-		collection.encodeBuffer.Reset()
+		encodeBuffer.Reset()
 		err = encoder.Encode(*p.files)
 		if err != nil {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xF"), collection.encodeBuffer.Bytes())
+		err = collection.db.Put(p.Key("xF"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
 	}
 
 	if p.deps != nil {
-		collection.encodeBuffer.Reset()
+		encodeBuffer.Reset()
 		err = encoder.Encode(*p.deps)
 		if err != nil {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xD"), collection.encodeBuffer.Bytes())
+		err = collection.db.Put(p.Key("xD"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
@@ -208,13 +245,13 @@ func (collection *PackageCollection) Update(p *Package) error {
 	}
 
 	if p.extra != nil {
-		collection.encodeBuffer.Reset()
+		encodeBuffer.Reset()
 		err = encoder.Encode(*p.extra)
 		if err != nil {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xE"), collection.encodeBuffer.Bytes())
+		err = collection.db.Put(p.Key("xE"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
@@ -245,7 +282,7 @@ func (collection *PackageCollection) DeleteByKey(key []byte) error {
 
 // Scan does full scan on all the packages
 func (collection *PackageCollection) Scan(q PackageQuery) (result *PackageList) {
-	result = NewPackageList()
+	result = NewPackageListWithDuplicates(true, 0)
 
 	for _, key := range collection.db.KeysByPrefix([]byte("P")) {
 		pkg, err := collection.ByKey(key)

deb/package_collection_test.go

@@ -3,7 +3,8 @@ package deb
 import (
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/utils"
-	. "launchpad.net/gocheck"
+
+	. "gopkg.in/check.v1"
 )
 
 type PackageCollectionSuite struct {

deb/package_deps.go

@@ -22,6 +22,12 @@ func parseDependencies(input Stanza, key string) []string {
 
 	delete(input, key)
 
+	value = strings.TrimSpace(value)
+	if value == "" {
+		// empty line is no depdencies
+		return nil
+	}
+
 	result := strings.Split(value, ",")
 	for i := range result {
 		result[i] = strings.TrimSpace(result[i])

deb/package_files.go

@@ -2,12 +2,15 @@ package deb
 
 import (
 	"encoding/binary"
+	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
 	"hash/fnv"
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
+	"strings"
 )
 
 // PackageFile is a single file entry in package
@@ -76,3 +79,71 @@ func (files PackageFiles) Swap(i, j int) {
 func (files PackageFiles) Less(i, j int) bool {
 	return files[i].Filename < files[j].Filename
 }
+
+func (files PackageFiles) parseSumField(input string, setter func(sum *utils.ChecksumInfo, data string)) (PackageFiles, error) {
+	for _, line := range strings.Split(input, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		parts := strings.Fields(line)
+
+		if len(parts) < 3 {
+			return nil, fmt.Errorf("unparseable hash sum line: %#v", line)
+		}
+
+		size, err := strconv.ParseInt(parts[1], 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("unable to parse size: %s", err)
+		}
+
+		filename := filepath.Base(parts[len(parts)-1])
+
+		found := false
+		pos := 0
+		for i, file := range files {
+			if file.Filename == filename {
+				found = true
+				pos = i
+				break
+			}
+		}
+
+		if !found {
+			files = append(files, PackageFile{Filename: filename})
+			pos = len(files) - 1
+		}
+
+		files[pos].Checksums.Size = size
+		setter(&files[pos].Checksums, parts[0])
+	}
+
+	return files, nil
+}
+
+// ParseSumFields populates PackageFiles by parsing stanza checksums fields
+func (files PackageFiles) ParseSumFields(stanza Stanza) (PackageFiles, error) {
+	var err error
+
+	files, err = files.parseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	files, err = files.parseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	files, err = files.parseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	files, err = files.parseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	return files, nil
+}

deb/package_files_test.go

@@ -3,9 +3,10 @@ package deb
 import (
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/utils"
-	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
+
+	. "gopkg.in/check.v1"
 )
 
 type PackageFilesSuite struct {

deb/package_test.go

@@ -4,10 +4,11 @@ import (
 	"bytes"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/utils"
-	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
 	"regexp"
+
+	. "gopkg.in/check.v1"
 )
 
 type PackageSuite struct {
@@ -21,7 +22,7 @@ func (s *PackageSuite) SetUpTest(c *C) {
 	s.stanza = packageStanza.Copy()
 
 	buf := bytes.NewBufferString(sourcePackageMeta)
-	s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza()
+	s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza(false)
 }
 
 func (s *PackageSuite) TestNewFromPara(c *C) {
@@ -42,7 +43,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 }
 
 func (s *PackageSuite) TestNewUdebFromPara(c *C) {
-	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
 	p := NewUdebPackageFromControlFile(stanza)
 
 	c.Check(p.IsSource, Equals, false)
@@ -124,6 +125,10 @@ func (s *PackageSuite) TestStanza(c *C) {
 	p := NewPackageFromControlFile(s.stanza.Copy())
 	stanza := p.Stanza()
 
+	for k := range s.stanza {
+		c.Check(stanza[k], Equals, s.stanza[k])
+	}
+
 	c.Assert(stanza, DeepEquals, s.stanza)
 
 	p, _ = NewSourcePackageFromControlFile(s.sourceStanza.Copy())
@@ -151,7 +156,7 @@ func (s *PackageSuite) TestGetField(c *C) {
 
 	p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
 
-	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
 	p5 := NewUdebPackageFromControlFile(stanza5)
 
 	c.Check(p.GetField("$Source"), Equals, "alien-arena")
@@ -398,7 +403,7 @@ func (s *PackageSuite) TestDownloadList(c *C) {
 	list, err := p.DownloadList(packagePool)
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []PackageDownloadTask{
-		PackageDownloadTask{
+		{
 			RepoURI:         "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb",
 			DestinationPath: poolPath,
 			Checksums: utils.ChecksumInfo{Size: 5,
@@ -444,7 +449,7 @@ func (s *PackageSuite) TestVerifyFiles(c *C) {
 	c.Check(result, Equals, true)
 }
 
-var packageStanza = Stanza{"Source": "alien-arena", "Pre-Depends": "dpkg (>= 1.6)", "Suggests": "alien-arena-mars", "Recommends": "aliean-arena-luna", "Depends": "libc6 (>= 2.7), alien-arena-data (>= 7.40)", "Filename": "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", "SHA1": " 46955e48cad27410a83740a21d766ce362364024", "SHA256": " eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", "Priority": "extra", "Maintainer": "Debian Games Team <pkg-games-devel@lists.alioth.debian.org>", "Description": "Common files for Alien Arena client and server ALIEN ARENA is a standalone 3D first person online deathmatch shooter\n crafted from the original source code of Quake II and Quake III, released\n by id Software under the GPL license. With features including 32 bit\n graphics, new particle engine and effects, light blooms, reflective water,\n hi resolution textures and skins, hi poly models, stain maps, ALIEN ARENA\n pushes the envelope of graphical beauty rivaling today's top games.\n .\n This package installs the common files for Alien Arena.\n", "Homepage": "http://red.planetarena.org", "Tag": "role::app-data, role::shared-lib, special::auto-inst-parts", "Installed-Size": "456", "Version": "7.40-2", "Replaces": "alien-arena (<< 7.33-1)", "Size": "187518", "MD5sum": "1e8cba92c41420aa7baa8a5718d67122", "Package": "alien-arena-common", "Section": "contrib/games", "Architecture": "i386"}
+var packageStanza = Stanza{"Source": "alien-arena", "Pre-Depends": "dpkg (>= 1.6)", "Suggests": "alien-arena-mars", "Recommends": "aliean-arena-luna", "Depends": "libc6 (>= 2.7), alien-arena-data (>= 7.40)", "Filename": "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", "SHA1": "46955e48cad27410a83740a21d766ce362364024", "SHA256": "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", "Priority": "extra", "Maintainer": "Debian Games Team <pkg-games-devel@lists.alioth.debian.org>", "Description": "Common files for Alien Arena client and server ALIEN ARENA is a standalone 3D first person online deathmatch shooter\n crafted from the original source code of Quake II and Quake III, released\n by id Software under the GPL license. With features including 32 bit\n graphics, new particle engine and effects, light blooms, reflective water,\n hi resolution textures and skins, hi poly models, stain maps, ALIEN ARENA\n pushes the envelope of graphical beauty rivaling today's top games.\n .\n This package installs the common files for Alien Arena.\n", "Homepage": "http://red.planetarena.org", "Tag": "role::app-data, role::shared-lib, special::auto-inst-parts", "Installed-Size": "456", "Version": "7.40-2", "Replaces": "alien-arena (<< 7.33-1)", "Size": "187518", "MD5sum": "1e8cba92c41420aa7baa8a5718d67122", "Package": "alien-arena-common", "Section": "contrib/games", "Architecture": "i386"}
 
 const sourcePackageMeta = `Package: access-modifier-checker
 Binary: libaccess-modifier-checker-java, libaccess-modifier-checker-java-doc

deb/ppa_test.go

@@ -2,7 +2,8 @@ package deb
 
 import (
 	"github.com/smira/aptly/utils"
-	. "launchpad.net/gocheck"
+
+	. "gopkg.in/check.v1"
 )
 
 type PpaSuite struct {

deb/publish.go

@@ -1,12 +1,14 @@
 package deb
 
 import (
+	"bufio"
 	"bytes"
-	"code.google.com/p/go-uuid/uuid"
+	"encoding/json"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
 	"io/ioutil"
 	"log"
@@ -14,6 +16,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -40,6 +43,8 @@ type PublishedRepo struct {
 	Architectures []string
 	// SourceKind is "local"/"repo"
 	SourceKind string
+	// Skip contents generation
+	SkipContents bool
 
 	// Map of sources by each component: component name -> source UUID
 	Sources map[string]string
@@ -56,6 +61,21 @@ type PublishedRepo struct {
 	rePublishing bool
 }
 
+// ParsePrefix splits [storage:]prefix into components
+func ParsePrefix(param string) (storage, prefix string) {
+	i := strings.LastIndex(param, ":")
+	if i != -1 {
+		storage = param[:i]
+		prefix = param[i+1:]
+		if prefix == "" {
+			prefix = "."
+		}
+	} else {
+		prefix = param
+	}
+	return
+}
+
 // walkUpTree goes from source in the tree of source snapshots/mirrors/local repos
 // gathering information about declared components and distributions
 func walkUpTree(source interface{}, collectionFactory *CollectionFactory) (rootDistributions []string, rootComponents []string) {
@@ -239,7 +259,42 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 	return result, nil
 }
 
-// String returns human-readable represenation of PublishedRepo
+// MarshalJSON requires object to be "loeaded completely"
+func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
+	type sourceInfo struct {
+		Component, Name string
+	}
+
+	sources := []sourceInfo{}
+	for component, item := range p.sourceItems {
+		name := ""
+		if item.snapshot != nil {
+			name = item.snapshot.Name
+		} else if item.localRepo != nil {
+			name = item.localRepo.Name
+		} else {
+			panic("no snapshot/local repo")
+		}
+		sources = append(sources, sourceInfo{
+			Component: component,
+			Name:      name,
+		})
+	}
+
+	return json.Marshal(map[string]interface{}{
+		"Architectures": p.Architectures,
+		"Distribution":  p.Distribution,
+		"Label":         p.Label,
+		"Origin":        p.Origin,
+		"Prefix":        p.Prefix,
+		"SourceKind":    p.SourceKind,
+		"Sources":       sources,
+		"Storage":       p.Storage,
+		"SkipContents":  p.SkipContents,
+	})
+}
+
+// String returns human-readable representation of PublishedRepo
 func (p *PublishedRepo) String() string {
 	var sources = []string{}
 
@@ -471,7 +526,11 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			progress.InitBar(int64(list.Len()), false)
 		}
 
-		err = list.ForEach(func(pkg *Package) error {
+		list.PrepareIndex()
+
+		contentIndexes := map[string]*ContentsIndex{}
+
+		err = list.ForEachIndexed(func(pkg *Package) error {
 			if progress != nil {
 				progress.AddBar(1)
 			}
@@ -494,12 +553,27 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 			for _, arch := range p.Architectures {
 				if pkg.MatchesArchitecture(arch) {
-					bufWriter, err := indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
+					var bufWriter *bufio.Writer
+
+					if !p.SkipContents {
+						key := fmt.Sprintf("%s-%v", arch, pkg.IsUdeb)
+
+						contentIndex := contentIndexes[key]
+
+						if contentIndex == nil {
+							contentIndex = NewContentsIndex()
+							contentIndexes[key] = contentIndex
+						}
+
+						contentIndex.Push(pkg, packagePool)
+					}
+
+					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
 					if err != nil {
 						return err
 					}
 
-					err = pkg.Stanza().WriteTo(bufWriter)
+					err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false)
 					if err != nil {
 						return err
 					}
@@ -513,6 +587,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			pkg.files = nil
 			pkg.deps = nil
 			pkg.extra = nil
+			pkg.contents = nil
 
 			return nil
 		})
@@ -521,6 +596,25 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			return fmt.Errorf("unable to process packages: %s", err)
 		}
 
+		for _, arch := range p.Architectures {
+			for _, udeb := range []bool{true, false} {
+				index := contentIndexes[fmt.Sprintf("%s-%v", arch, udeb)]
+				if index == nil || index.Empty() {
+					continue
+				}
+
+				bufWriter, err := indexes.ContentsIndex(component, arch, udeb).BufWriter()
+				if err != nil {
+					return fmt.Errorf("unable to generate contents index: %v", err)
+				}
+
+				_, err = index.WriteTo(bufWriter)
+				if err != nil {
+					return fmt.Errorf("unable to generate contents index: %v", err)
+				}
+			}
+		}
+
 		if progress != nil {
 			progress.ShutdownBar()
 		}
@@ -545,9 +639,13 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				release["Origin"] = p.GetOrigin()
 				release["Label"] = p.GetLabel()
 
-				bufWriter, err := indexes.ReleaseIndex(component, arch, udeb).BufWriter()
+				var bufWriter *bufio.Writer
+				bufWriter, err = indexes.ReleaseIndex(component, arch, udeb).BufWriter()
+				if err != nil {
+					return fmt.Errorf("unable to get ReleaseIndex writer: %s", err)
+				}
 
-				err = release.WriteTo(bufWriter)
+				err = release.WriteTo(bufWriter, false, true)
 				if err != nil {
 					return fmt.Errorf("unable to create Release file: %s", err)
 				}
@@ -567,13 +665,15 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	release := make(Stanza)
 	release["Origin"] = p.GetOrigin()
 	release["Label"] = p.GetLabel()
+	release["Suite"] = p.Distribution
 	release["Codename"] = p.Distribution
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{"source"}), " ")
 	release["Description"] = " Generated by aptly\n"
-	release["MD5Sum"] = "\n"
+	release["MD5Sum"] = ""
-	release["SHA1"] = "\n"
+	release["SHA1"] = ""
-	release["SHA256"] = "\n"
+	release["SHA256"] = ""
+	release["SHA512"] = ""
 
 	release["Components"] = strings.Join(p.Components(), " ")
 
@@ -581,6 +681,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		release["MD5Sum"] += fmt.Sprintf(" %s %8d %s\n", info.MD5, info.Size, path)
 		release["SHA1"] += fmt.Sprintf(" %s %8d %s\n", info.SHA1, info.Size, path)
 		release["SHA256"] += fmt.Sprintf(" %s %8d %s\n", info.SHA256, info.Size, path)
+		release["SHA512"] += fmt.Sprintf(" %s %8d %s\n", info.SHA512, info.Size, path)
 	}
 
 	releaseFile := indexes.ReleaseFile()
@@ -589,7 +690,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		return err
 	}
 
-	err = release.WriteTo(bufWriter)
+	err = release.WriteTo(bufWriter, false, true)
 	if err != nil {
 		return fmt.Errorf("unable to create Release file: %s", err)
 	}
@@ -648,6 +749,7 @@ func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStor
 
 // PublishedRepoCollection does listing, updating/adding/deleting of PublishedRepos
 type PublishedRepoCollection struct {
+	*sync.RWMutex
 	db   database.Storage
 	list []*PublishedRepo
 }
@@ -655,7 +757,8 @@ type PublishedRepoCollection struct {
 // NewPublishedRepoCollection loads PublishedRepos from DB and makes up collection
 func NewPublishedRepoCollection(db database.Storage) *PublishedRepoCollection {
 	result := &PublishedRepoCollection{
-		db: db,
+		RWMutex: &sync.RWMutex{},
+		db:      db,
 	}
 
 	blobs := db.FetchByPrefix([]byte("U"))
@@ -940,7 +1043,8 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
 
 // Remove removes published repository, cleaning up directories, files
 func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly.PublishedStorageProvider,
-	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
+	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress,
+	force bool) error {
 	repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return err
@@ -981,7 +1085,9 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 		err = collection.CleanupPrefixComponentFiles(repo.Prefix, cleanComponents,
 			publishedStorageProvider.GetPublishedStorage(storage), collectionFactory, progress)
 		if err != nil {
-			return err
+			if !force {
+				return fmt.Errorf("cleanup failed, use -force-drop to override: %s", err)
+			}
 		}
 	}
 

deb/publish_test.go

@@ -9,9 +9,10 @@ import (
 	"github.com/smira/aptly/files"
 	"github.com/ugorji/go/codec"
 	"io/ioutil"
-	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
+
+	. "gopkg.in/check.v1"
 )
 
 type pathExistsChecker struct {
@@ -36,6 +37,9 @@ func (n *NullSigner) Init() error {
 func (n *NullSigner) SetKey(keyRef string) {
 }
 
+func (n *NullSigner) SetBatch(batch bool) {
+}
+
 func (n *NullSigner) SetKeyRing(keyring, secretKeyring string) {
 }
 
@@ -113,14 +117,19 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.packageCollection.Update(s.p3)
 
 	s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	s.repo.SkipContents = true
 
 	s.repo2, _ = NewPublishedRepo("", "ppa", "maverick", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo2.SkipContents = true
 
 	s.repo3, _ = NewPublishedRepo("", "linux", "natty", nil, []string{"main", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	s.repo3.SkipContents = true
 
 	s.repo4, _ = NewPublishedRepo("", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo4.SkipContents = true
 
 	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo5.SkipContents = true
 
 	poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
 	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
@@ -296,7 +305,7 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	c.Assert(err, IsNil)
 
 	cfr := NewControlFileReader(rf)
-	st, err := cfr.ReadStanza()
+	st, err := cfr.ReadStanza(true)
 	c.Assert(err, IsNil)
 
 	c.Check(st["Origin"], Equals, "ppa squeeze")
@@ -309,13 +318,13 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	cfr = NewControlFileReader(pf)
 
 	for i := 0; i < 3; i++ {
-		st, err = cfr.ReadStanza()
+		st, err = cfr.ReadStanza(false)
 		c.Assert(err, IsNil)
 
 		c.Check(st["Filename"], Equals, "pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
 	}
 
-	st, err = cfr.ReadStanza()
+	st, err = cfr.ReadStanza(false)
 	c.Assert(err, IsNil)
 	c.Assert(st, IsNil)
 
@@ -323,7 +332,7 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	c.Assert(err, IsNil)
 
 	cfr = NewControlFileReader(drf)
-	st, err = cfr.ReadStanza()
+	st, err = cfr.ReadStanza(true)
 	c.Assert(err, IsNil)
 
 	c.Check(st["Archive"], Equals, "squeeze")
@@ -736,7 +745,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
-	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
@@ -756,10 +765,10 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 
-	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil)
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
 	c.Check(err, ErrorMatches, ".*not found")
 
-	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil)
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false)
 	c.Check(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
@@ -774,7 +783,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
-	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil)
+	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda")
@@ -796,7 +805,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) {
-	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil)
+	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")

deb/query.go

@@ -7,6 +7,16 @@ import (
 	"strings"
 )
 
+// PackageLike is something like Package :) To be refined later
+type PackageLike interface {
+	GetField(string) string
+	MatchesDependency(Dependency) bool
+	MatchesArchitecture(string) bool
+	GetName() string
+	GetVersion() string
+	GetArchitecture() string
+}
+
 // PackageCatalog is abstraction on top of PackageCollection and PackageList
 type PackageCatalog interface {
 	Scan(q PackageQuery) (result *PackageList)
@@ -18,7 +28,7 @@ type PackageCatalog interface {
 // PackageQuery is interface of predicate on Package
 type PackageQuery interface {
 	// Matches calculates match of condition against package
-	Matches(pkg *Package) bool
+	Matches(pkg PackageLike) bool
 	// Fast returns if search strategy is possible for this query
 	Fast(list PackageCatalog) bool
 	// Query performs search on package list
@@ -63,7 +73,7 @@ type DependencyQuery struct {
 }
 
 // Matches if any of L, R matches
-func (q *OrQuery) Matches(pkg *Package) bool {
+func (q *OrQuery) Matches(pkg PackageLike) bool {
 	return q.L.Matches(pkg) || q.R.Matches(pkg)
 }
 
@@ -89,7 +99,7 @@ func (q *OrQuery) String() string {
 }
 
 // Matches if both of L, R matches
-func (q *AndQuery) Matches(pkg *Package) bool {
+func (q *AndQuery) Matches(pkg PackageLike) bool {
 	return q.L.Matches(pkg) && q.R.Matches(pkg)
 }
 
@@ -120,7 +130,7 @@ func (q *AndQuery) String() string {
 }
 
 // Matches if not matches
-func (q *NotQuery) Matches(pkg *Package) bool {
+func (q *NotQuery) Matches(pkg PackageLike) bool {
 	return !q.Q.Matches(pkg)
 }
 
@@ -141,9 +151,9 @@ func (q *NotQuery) String() string {
 }
 
 // Matches on generic field
-func (q *FieldQuery) Matches(pkg *Package) bool {
+func (q *FieldQuery) Matches(pkg PackageLike) bool {
 	if q.Field == "$Version" {
-		return pkg.MatchesDependency(Dependency{Pkg: pkg.Name, Relation: q.Relation, Version: q.Value, Regexp: q.Regexp})
+		return pkg.MatchesDependency(Dependency{Pkg: pkg.GetName(), Relation: q.Relation, Version: q.Value, Regexp: q.Regexp})
 	}
 	if q.Field == "$Architecture" && q.Relation == VersionEqual {
 		return pkg.MatchesArchitecture(q.Value)
@@ -218,7 +228,7 @@ func (q *FieldQuery) String() string {
 }
 
 // Matches on dependency condition
-func (q *DependencyQuery) Matches(pkg *Package) bool {
+func (q *DependencyQuery) Matches(pkg PackageLike) bool {
 	return pkg.MatchesDependency(q.Dep)
 }
 
@@ -247,8 +257,8 @@ func (q *DependencyQuery) String() string {
 }
 
 // Matches on specific properties
-func (q *PkgQuery) Matches(pkg *Package) bool {
+func (q *PkgQuery) Matches(pkg PackageLike) bool {
-	return pkg.Name == q.Pkg && pkg.Version == q.Version && pkg.Architecture == q.Arch
+	return pkg.GetName() == q.Pkg && pkg.GetVersion() == q.Version && pkg.GetArchitecture() == q.Arch
 }
 
 // Fast is always true for package query

deb/reflist.go

@@ -2,6 +2,8 @@ package deb
 
 import (
 	"bytes"
+	"encoding/json"
+	"github.com/AlekSi/pointer"
 	"github.com/ugorji/go/codec"
 	"sort"
 )
@@ -92,6 +94,21 @@ func (l *PackageRefList) Has(p *Package) bool {
 	return i < len(l.Refs) && bytes.Compare(l.Refs[i], key) == 0
 }
 
+// Strings builds list of strings with package keys
+func (l *PackageRefList) Strings() []string {
+	if l == nil {
+		return []string{}
+	}
+
+	result := make([]string, l.Len())
+
+	for i := 0; i < l.Len(); i++ {
+		result[i] = string(l.Refs[i])
+	}
+
+	return result
+}
+
 // Substract returns all packages in l that are not in r
 func (l *PackageRefList) Substract(r *PackageRefList) *PackageRefList {
 	result := &PackageRefList{Refs: make([][]byte, 0, 128)}
@@ -139,6 +156,27 @@ type PackageDiff struct {
 	Left, Right *Package
 }
 
+// Check interface
+var (
+	_ json.Marshaler = PackageDiff{}
+)
+
+// MarshalJSON implements json.Marshaler interface
+func (d PackageDiff) MarshalJSON() ([]byte, error) {
+	serialized := struct {
+		Left, Right *string
+	}{}
+
+	if d.Left != nil {
+		serialized.Left = pointer.ToString(string(d.Left.Key("")))
+	}
+	if d.Right != nil {
+		serialized.Right = pointer.ToString(string(d.Right.Key("")))
+	}
+
+	return json.Marshal(serialized)
+}
+
 // PackageDiffs is a list of PackageDiff records
 type PackageDiffs []PackageDiff
 
@@ -232,8 +270,9 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle
 
 // Merge merges reflist r into current reflist. If overrideMatching, merge
 // replaces matching packages (by architecture/name) with reference from r.
-// Otherwise, all packages are saved.
+// If ignoreConflicting is set, all packages are preserved, otherwise conflciting
-func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result *PackageRefList) {
+// packages are overwritten with packages from "right" snapshot.
+func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching, ignoreConflicting bool) (result *PackageRefList) {
 	var overriddenArch, overridenName []byte
 
 	// pointer to left and right reflists
@@ -270,13 +309,23 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result
 			overridenName = nil
 			overriddenArch = nil
 		} else {
+			partsL := bytes.Split(rl, []byte(" "))
+			archL, nameL, versionL := partsL[0][1:], partsL[1], partsL[2]
+
+			partsR := bytes.Split(rr, []byte(" "))
+			archR, nameR, versionR := partsR[0][1:], partsR[1], partsR[2]
+
+			if !ignoreConflicting && bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) && bytes.Equal(versionL, versionR) {
+				// conflicting duplicates with same arch, name, version, but different file hash
+				result.Refs = append(result.Refs, r.Refs[ir])
+				il++
+				ir++
+				overridenName = nil
+				overriddenArch = nil
+				continue
+			}
+
 			if overrideMatching {
-				partsL := bytes.Split(rl, []byte(" "))
-				archL, nameL := partsL[0][1:], partsL[1]
-
-				partsR := bytes.Split(rr, []byte(" "))
-				archR, nameR := partsR[0][1:], partsR[1]
-
 				if bytes.Equal(archL, overriddenArch) && bytes.Equal(nameL, overridenName) {
 					// this package has already been overriden on the right
 					il++
@@ -314,15 +363,15 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result
 // packages and reduces it to only the latest of each package. The operations
 // are done in-place. This implements a "latest wins" approach which can be used
 // while merging two or more snapshots together.
-func FilterLatestRefs(r *PackageRefList) {
+func (l *PackageRefList) FilterLatestRefs() {
 	var (
 		lastArch, lastName, lastVer []byte
 		arch, name, ver             []byte
 		parts                       [][]byte
 	)
 
-	for i := 0; i < len(r.Refs); i++ {
+	for i := 0; i < len(l.Refs); i++ {
-		parts = bytes.Split(r.Refs[i][1:], []byte(" "))
+		parts = bytes.Split(l.Refs[i][1:], []byte(" "))
 		arch, name, ver = parts[0], parts[1], parts[2]
 
 		if bytes.Equal(arch, lastArch) && bytes.Equal(name, lastName) {
@@ -332,10 +381,10 @@ func FilterLatestRefs(r *PackageRefList) {
 			// Remove the older refs from the result
 			if vres > 0 {
 				// ver[i] > ver[i-1], remove element i-1
-				r.Refs = append(r.Refs[:i-1], r.Refs[i:]...)
+				l.Refs = append(l.Refs[:i-1], l.Refs[i:]...)
 			} else {
 				// ver[i] < ver[i-1], remove element i
-				r.Refs = append(r.Refs[:i], r.Refs[i+1:]...)
+				l.Refs = append(l.Refs[:i], l.Refs[i+1:]...)
 				arch, name, ver = lastArch, lastName, lastVer
 			}
 

deb/reflist_test.go

@@ -3,7 +3,8 @@ package deb
 import (
 	"errors"
 	"github.com/smira/aptly/database"
-	. "launchpad.net/gocheck"
+
+	. "gopkg.in/check.v1"
 )
 
 type PackageRefListSuite struct {
@@ -168,13 +169,13 @@ func (s *PackageRefListSuite) TestDiff(c *C) {
 	coll := NewPackageCollection(db)
 
 	packages := []*Package{
-		&Package{Name: "lib", Version: "1.0", Architecture: "i386"},      //0
+		{Name: "lib", Version: "1.0", Architecture: "i386"},      //0
-		&Package{Name: "dpkg", Version: "1.7", Architecture: "i386"},     //1
+		{Name: "dpkg", Version: "1.7", Architecture: "i386"},     //1
-		&Package{Name: "data", Version: "1.1~bp1", Architecture: "all"},  //2
+		{Name: "data", Version: "1.1~bp1", Architecture: "all"},  //2
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //3
+		{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //3
-		&Package{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //4
+		{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //4
-		&Package{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5
+		{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5
-		&Package{Name: "xyz", Version: "3.0", Architecture: "sparc"},     //6
+		{Name: "xyz", Version: "3.0", Architecture: "sparc"},     //6
 	}
 
 	for _, p := range packages {
@@ -240,17 +241,20 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	coll := NewPackageCollection(db)
 
 	packages := []*Package{
-		&Package{Name: "lib", Version: "1.0", Architecture: "i386"},      //0
+		{Name: "lib", Version: "1.0", Architecture: "i386"},                      //0
-		&Package{Name: "dpkg", Version: "1.7", Architecture: "i386"},     //1
+		{Name: "dpkg", Version: "1.7", Architecture: "i386"},                     //1
-		&Package{Name: "data", Version: "1.1~bp1", Architecture: "all"},  //2
+		{Name: "data", Version: "1.1~bp1", Architecture: "all"},                  //2
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //3
+		{Name: "app", Version: "1.1~bp1", Architecture: "i386"},                  //3
-		&Package{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //4
+		{Name: "app", Version: "1.1~bp2", Architecture: "i386"},                  //4
-		&Package{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5
+		{Name: "app", Version: "1.1~bp2", Architecture: "amd64"},                 //5
-		&Package{Name: "dpkg", Version: "1.0", Architecture: "i386"},     //6
+		{Name: "dpkg", Version: "1.0", Architecture: "i386"},                     //6
-		&Package{Name: "xyz", Version: "1.0", Architecture: "sparc"},     //7
+		{Name: "xyz", Version: "1.0", Architecture: "sparc"},                     //7
+		{Name: "dpkg", Version: "1.0", Architecture: "i386", FilesHash: 0x34445}, //8
+		{Name: "app", Version: "1.1~bp2", Architecture: "i386", FilesHash: 0x44}, //9
 	}
 
 	for _, p := range packages {
+		p.V06Plus = true
 		coll.Update(p)
 	}
 
@@ -268,35 +272,67 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 	listB.Add(packages[5])
 	listB.Add(packages[6])
 
+	listC := NewPackageList()
+	listC.Add(packages[0])
+	listC.Add(packages[8])
+	listC.Add(packages[9])
+
 	reflistA := NewPackageRefListFromPackageList(listA)
 	reflistB := NewPackageRefListFromPackageList(listB)
+	reflistC := NewPackageRefListFromPackageList(listC)
 
-	mergeAB := reflistA.Merge(reflistB, true)
+	mergeAB := reflistA.Merge(reflistB, true, false)
-	mergeBA := reflistB.Merge(reflistA, true)
+	mergeBA := reflistB.Merge(reflistA, true, false)
+	mergeAC := reflistA.Merge(reflistC, true, false)
+	mergeBC := reflistB.Merge(reflistC, true, false)
+	mergeCB := reflistC.Merge(reflistB, true, false)
 
 	c.Check(toStrSlice(mergeAB), DeepEquals,
-		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 lib 1.0", "Psparc xyz 1.0"})
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 dpkg 1.0 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
 	c.Check(toStrSlice(mergeBA), DeepEquals,
-		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"})
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp1 00000000", "Pi386 dpkg 1.7 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
+	c.Check(toStrSlice(mergeAC), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
+	c.Check(toStrSlice(mergeBC), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000"})
+	c.Check(toStrSlice(mergeCB), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 dpkg 1.0 00000000", "Pi386 lib 1.0 00000000"})
 
-	mergeABall := reflistA.Merge(reflistB, false)
+	mergeABall := reflistA.Merge(reflistB, false, false)
-	mergeBAall := reflistB.Merge(reflistA, false)
+	mergeBAall := reflistB.Merge(reflistA, false, false)
+	mergeACall := reflistA.Merge(reflistC, false, false)
+	mergeBCall := reflistB.Merge(reflistC, false, false)
+	mergeCBall := reflistC.Merge(reflistB, false, false)
 
 	c.Check(mergeABall, DeepEquals, mergeBAall)
 	c.Check(toStrSlice(mergeBAall), DeepEquals,
-		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"})
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp1 00000000", "Pi386 app 1.1~bp2 00000000",
+			"Pi386 dpkg 1.0 00000000", "Pi386 dpkg 1.7 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
+
+	c.Check(mergeBCall, Not(DeepEquals), mergeCBall)
+	c.Check(toStrSlice(mergeACall), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pi386 app 1.1~bp1 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445",
+			"Pi386 dpkg 1.7 00000000", "Pi386 lib 1.0 00000000", "Psparc xyz 1.0 00000000"})
+	c.Check(toStrSlice(mergeBCall), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044", "Pi386 dpkg 1.0 00034445",
+			"Pi386 lib 1.0 00000000"})
+
+	mergeBCwithConflicts := reflistB.Merge(reflistC, false, true)
+	c.Check(toStrSlice(mergeBCwithConflicts), DeepEquals,
+		[]string{"Pall data 1.1~bp1 00000000", "Pamd64 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000000", "Pi386 app 1.1~bp2 00000044",
+			"Pi386 dpkg 1.0 00000000", "Pi386 dpkg 1.0 00034445", "Pi386 lib 1.0 00000000"})
 }
 
 func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) {
 	packages := []*Package{
-		&Package{Name: "lib", Version: "1.0", Architecture: "i386"},
+		{Name: "lib", Version: "1.0", Architecture: "i386"},
-		&Package{Name: "lib", Version: "1.2~bp1", Architecture: "i386"},
+		{Name: "lib", Version: "1.2~bp1", Architecture: "i386"},
-		&Package{Name: "lib", Version: "1.2", Architecture: "i386"},
+		{Name: "lib", Version: "1.2", Architecture: "i386"},
-		&Package{Name: "dpkg", Version: "1.2", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.2", Architecture: "i386"},
-		&Package{Name: "dpkg", Version: "1.3", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.3", Architecture: "i386"},
-		&Package{Name: "dpkg", Version: "1.3~bp2", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.3~bp2", Architecture: "i386"},
-		&Package{Name: "dpkg", Version: "1.5", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.5", Architecture: "i386"},
-		&Package{Name: "dpkg", Version: "1.6", Architecture: "i386"},
+		{Name: "dpkg", Version: "1.6", Architecture: "i386"},
 	}
 
 	rl := NewPackageList()
@@ -310,7 +346,7 @@ func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) {
 	rl.Add(packages[7])
 
 	result := NewPackageRefListFromPackageList(rl)
-	FilterLatestRefs(result)
+	result.FilterLatestRefs()
 
 	c.Check(toStrSlice(result), DeepEquals,
 		[]string{"Pi386 dpkg 1.6", "Pi386 lib 1.2"})

deb/remote.go

@@ -2,20 +2,22 @@ package deb
 
 import (
 	"bytes"
-	"code.google.com/p/go-uuid/uuid"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/http"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
 	"log"
 	"net/url"
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 )
@@ -56,6 +58,8 @@ type RemoteRepo struct {
 	Filter string
 	// FilterWithDeps to include dependencies from filter query
 	FilterWithDeps bool
+	// SkipComponentCheck skips component list verification
+	SkipComponentCheck bool
 	// Status marks state of repository (being updated, no action)
 	Status int
 	// WorkerPID is PID of the process modifying the mirror (if any)
@@ -142,7 +146,7 @@ func (repo *RemoteRepo) IsFlat() bool {
 	return repo.Distribution == "" || (strings.HasPrefix(repo.Distribution, ".") && strings.HasSuffix(repo.Distribution, "/"))
 }
 
-// NumPackages return number of packages retrived from remote repo
+// NumPackages return number of packages retrieved from remote repo
 func (repo *RemoteRepo) NumPackages() int {
 	if repo.packageRefs == nil {
 		return 0
@@ -259,7 +263,7 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error
 		}
 		defer inrelease.Close()
 
-		err = verifier.VerifyClearsigned(inrelease)
+		_, err = verifier.VerifyClearsigned(inrelease, true)
 		if err != nil {
 			goto splitsignature
 		}
@@ -300,13 +304,16 @@ ok:
 	defer release.Close()
 
 	sreader := NewControlFileReader(release)
-	stanza, err := sreader.ReadStanza()
+	stanza, err := sreader.ReadStanza(true)
 	if err != nil {
 		return err
 	}
 
 	if !repo.IsFlat() {
 		architectures := strings.Split(stanza["Architectures"], " ")
+		sort.Strings(architectures)
+		// "source" architecture is never present, despite Release file claims
+		architectures = utils.StrSlicesSubstract(architectures, []string{"source"})
 		if len(repo.Architectures) == 0 {
 			repo.Architectures = architectures
 		} else {
@@ -318,14 +325,19 @@ ok:
 		}
 
 		components := strings.Split(stanza["Components"], " ")
-		for i := range components {
+		if strings.Contains(repo.Distribution, "/") {
-			components[i] = path.Base(components[i])
+			distributionLast := path.Base(repo.Distribution) + "/"
+			for i := range components {
+				if strings.HasPrefix(components[i], distributionLast) {
+					components[i] = components[i][len(distributionLast):]
+				}
+			}
 		}
 		if len(repo.Components) == 0 {
 			repo.Components = components
-		} else {
+		} else if !repo.SkipComponentCheck {
 			err = utils.StringsIsSubset(repo.Components, components,
-				fmt.Sprintf("component %%s not available in repo %s", repo))
+				fmt.Sprintf("component %%s not available in repo %s, use -force-components to override", repo))
 			if err != nil {
 				return err
 			}
@@ -380,6 +392,8 @@ ok:
 		return err
 	}
 
+	delete(stanza, "SHA512")
+
 	repo.Meta = stanza
 
 	return nil
@@ -429,7 +443,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 		sreader := NewControlFileReader(packagesReader)
 
 		for {
-			stanza, err := sreader.ReadStanza()
+			stanza, err := sreader.ReadStanza(false)
 			if err != nil {
 				return err
 			}
@@ -454,7 +468,11 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 			}
 			err = repo.packageList.Add(p)
 			if err != nil {
-				return err
+				if _, ok := err.(*PackageConflictError); ok {
+					progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
+				} else {
+					return err
+				}
 			}
 
 			err = collectionFactory.PackageCollection().Update(p)
@@ -593,6 +611,7 @@ func (repo *RemoteRepo) RefKey() []byte {
 
 // RemoteRepoCollection does listing, updating/adding/deleting of RemoteRepos
 type RemoteRepoCollection struct {
+	*sync.RWMutex
 	db   database.Storage
 	list []*RemoteRepo
 }
@@ -600,7 +619,8 @@ type RemoteRepoCollection struct {
 // NewRemoteRepoCollection loads RemoteRepos from DB and makes up collection
 func NewRemoteRepoCollection(db database.Storage) *RemoteRepoCollection {
 	result := &RemoteRepoCollection{
-		db: db,
+		RWMutex: &sync.RWMutex{},
+		db:      db,
 	}
 
 	blobs := db.FetchByPrefix([]byte("R"))