Fix temporary contents DB being left behind after publishing

NB: Go `defer` order execution is reverse to the order `defer` statements
are executed.

So before the change, `Drop()` was called before `Close()`, which was no-op.

Change that to explicit order in single func, print errors if they happen.

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,14 @@
+<!--- Provide a general summary of the issue in the Title above -->
+
+## Detailed Description
+<!--- Provide a detailed description of the change or addition you are proposing -->
+
+## Context
+<!--- Why is this change important to you? How would you use it? -->
+<!--- How can it benefit other users? -->
+
+## Possible Implementation
+<!--- Not obligatory, but suggest an idea for implementing addition or change -->
+
+## Your Environment
+<!--- Include as many relevant details about the environment you experienced the bug in -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,22 @@
+Fixes #
+
+## Requirements
+
+All new code should be covered with tests, documentation should be updated. CI should pass.
+
+## Description of the Change
+
+<!--
+
+Why this change is important?
+
+-->
+
+## Checklist
+
+- [ ] unit-test added (if change is algorithm)
+- [ ] functional test added/updated (if change is functional)
+- [ ] man page updated (if applicable)
+- [ ] bash completion updated (if applicable)
+- [ ] documentation updated
+- [ ] author name in `AUTHORS`

.gitignore

@@ -27,8 +27,10 @@ coverage*.out
 
 *.pyc
 
-_vendor/
+xc-out/
+root/
 
-gen
 man/aptly.1.html
-man/aptly.1.ronn
+man/aptly.1.ronn
+
+.goxc.local.json

.goxc.json

@@ -0,0 +1,46 @@
+{
+	"AppName": "aptly",
+	"ArtifactsDest": "xc-out/",
+	"TasksExclude": [
+		"rmbin",
+		"go-test",
+		"go-vet"
+	],
+    "TasksAppend": [
+		"bintray"
+    ],
+    "TaskSettings": {
+        "deb": {
+            "metadata": {
+                "maintainer": "Andrey Smirnov",
+                "maintainerEmail": "me@smira.ru",
+                "description": "Debian repository management tool"
+            },
+            "metadata-deb": {
+                "License": "MIT",
+                "Homepage": "https://www.aptly.info/",
+                "Recommends": "bzip2, graphviz, xz-utils",
+                "Depends": ""
+            },
+            "other-mapped-files": {
+                "/": "root/"
+            }
+        },
+        "bintray": {
+            "repository": "aptly",
+            "subject": "smira",
+            "package": "aptly",
+            "downloadspage": "bintray.md"
+        }
+    },
+	"ResourcesInclude": "README.rst,LICENSE,AUTHORS,man/aptly.1",
+	"Arch": "386 amd64",
+	"Os": "linux darwin freebsd",
+	"MainDirsExclude": "_man,vendor",
+	"BuildSettings": {
+		"LdFlagsXVars": {
+			"Version": "main.Version"
+		}
+	},
+	"ConfigVersion": "0.9"
+}

.travis.yml

@@ -1,31 +1,43 @@
+sudo: false
+
 language: go
 
 go:
-  - 1.3.3
+  - 1.6
+  - 1.7
+  - 1.8
   - tip
 
+addons:
+  apt:
+    packages:
+    - python-virtualenv
+    - graphviz
+
 env:
   global:
     - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
     - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
     - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
 before_install:
-  - sudo apt-get update -qq
-  - sudo apt-get install -y python-virtualenv graphviz
   - virtualenv env
   - . env/bin/activate
-  - pip install boto requests python-swiftclient
+  - pip install six packaging appdirs
+  - pip install -U pip setuptools
+  - pip install boto requests requests-unixsocket python-swiftclient
+  - mkdir -p $GOPATH/src/github.com/smira
+  - ln -s $TRAVIS_BUILD_DIR $GOPATH/src/github.com/smira || true
+  - cd $GOPATH/src/github.com/smira/aptly
+  - make version
 install:
   - make prepare
 
-
 script: make travis
 
 matrix:
     allow_failures:
         - go: tip
 
-
 notifications:
   webhooks:
     urls:

AUTHORS

@@ -15,3 +15,14 @@ List of contributors, in chronological order:
 * Michael Koval (https://github.com/mkoval)
 * Alexander Guy (https://github.com/alexanderguy)
 * Sebastien Badia (https://github.com/sbadia)
+* Szymon Sobik (https://github.com/sobczyk)
+* Paul Krohn (https://github.com/paul-krohn)
+* Vincent Bernat (https://github.com/vincentbernat)
+* x539 (https://github.com/x539)
+* Phil Frost (https://github.com/bitglue)
+* Benoit Foucher (https://github.com/bentoi)
+* Geoffrey Thomas (https://github.com/geofft)
+* Oliver Sauder (https://github.com/sliverc)
+* Harald Sitter (https://github.com/apachelogger)
+* Johannes Layher (https://github.com/jola5)
+* Charles Hsu (https://github.com/charz)

CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at [team@aptly.info](mailto:team@aptly.info). All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

Gomfile

@@ -1,33 +0,0 @@
-gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
-gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2'
-gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556'
-gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
-gom 'github.com/AlekSi/pointer', :commit => '5f6d527dae3d678b46fbb20331ddf44e2b841943'
-gom 'github.com/cheggaaa/pb', :commit => '2c1b74620cc58a81ac152ee2d322e28c806d81ed'
-gom 'github.com/gin-gonic/gin', :commit => 'b1758d3bfa09e61ddbc1c9a627e936eec6a170de'
-gom 'github.com/jlaffaye/ftp', :commit => 'fec71e62e457557fbe85cefc847a048d57815d76'
-gom 'github.com/julienschmidt/httprouter', :commit => '46807412fe50aaceb73bb57061c2230fd26a1640'
-gom 'github.com/mattn/go-shellwords', :commit => 'c7ca6f94add751566a61cf2199e1de78d4c3eee4'
-gom 'github.com/mitchellh/goamz/s3', :commit => 'e7664b32019f31fd1bdf33f9e85f28722f700405'
-gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
-gom 'github.com/ncw/swift', :commit => '384ef27c70645e285f8bb9d02276bf654d06027e'
-gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
-gom 'github.com/smira/flag', :commit => '357ed3e599ffcbd4aeaa828e1d10da2df3ea5107'
-gom 'github.com/smira/go-ftp-protocol/protocol', :commit => '066b75c2b70dca7ae10b1b88b47534a3c31ccfaa'
-gom 'github.com/syndtr/gosnappy/snappy', :commit => 'ce8acff4829e0c2458a67ead32390ac0a381c862'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => '97e257099d2ab9578151ba85e2641e2cd14d3ca8'
-gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
-gom 'github.com/vaughan0/go-ini', :commit => 'a98ad7ee00ec53921f08832bc06ecf7fd600e6a1'
-gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
-gom 'golang.org/x/crypto/ssh/terminal', :commit => 'a7ead6ddf06233883deca151dffaef2effbf498f'
-
-group :test do
-    gom 'gopkg.in/check.v1'
-end
-
-group :development do
-   gom 'github.com/golang/lint/golint'
-   gom 'github.com/mattn/goveralls'
-   gom 'github.com/axw/gocov/gocov'
-   gom 'golang.org/x/tools/cmd/cover'
-end

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2013-2014 Andrey Smirnov. All rights reserved.
+Copyright 2013-2015 aptly authors. All rights reserved.
 
 MIT License
 

Makefile

@@ -1,86 +1,84 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
+VERSION=$(shell git describe --tags | sed 's@^v@@' | sed 's@-@+@g')
 PACKAGES=context database deb files http query swift s3 utils
-ALL_PACKAGES=api aptly context cmd console database deb files http query swift s3 utils
-BINPATH=$(abspath ./_vendor/bin)
-GOM_ENVIRONMENT=-test
 PYTHON?=python
+TESTS?=
+BINPATH?=$(GOPATH)/bin
 
 ifeq ($(GOVERSION), devel)
 TRAVIS_TARGET=coveralls
-GOM_ENVIRONMENT+=-development
 else
 TRAVIS_TARGET=test
 endif
 
-ifeq ($(TRAVIS), true)
-GOM=$(HOME)/gopath/bin/gom
-else
-GOM=gom
-endif
-
 all: test check system-test
 
 prepare:
-	go get -u github.com/mattn/gom
-	$(GOM) $(GOM_ENVIRONMENT) install
+	go get -u github.com/mattn/goveralls
+	go get -u github.com/axw/gocov/gocov
+	go get -u golang.org/x/tools/cmd/cover
+	go get -u github.com/alecthomas/gometalinter
+	gometalinter --install
+
+dev:
+	go get -u github.com/golang/dep/...
+	go get -u github.com/laher/goxc
 
 coverage.out:
 	rm -f coverage.*.out
-	for i in $(PACKAGES); do $(GOM) test -coverprofile=coverage.$$i.out -covermode=count ./$$i; done
+	for i in $(PACKAGES); do go test -coverprofile=coverage.$$i.out -covermode=count ./$$i; done
 	echo "mode: count" > coverage.out
 	grep -v -h "mode: count" coverage.*.out >> coverage.out
 	rm -f coverage.*.out
 
 coverage: coverage.out
-	$(GOM) exec go tool cover -html=coverage.out
+	go tool cover -html=coverage.out
 	rm -f coverage.out
 
 check:
-	$(GOM) exec go tool vet -all=true $(ALL_PACKAGES:%=./%)
-	$(GOM) exec golint $(ALL_PACKAGES:%=./%)
+	gometalinter --vendor --vendored-linters --config=linter.json ./...
 
 install:
-	$(GOM) build -o $(BINPATH)/aptly
+	go install -v -ldflags "-X main.Version=$(VERSION)"
 
 system-test: install
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long
+	APTLY_VERSION=$(VERSION) PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long $(TESTS)
 
-travis: $(TRAVIS_TARGET) system-test
+travis: $(TRAVIS_TARGET) check system-test
 
 test:
-	$(GOM) test -v ./... -gocheck.v=true
+	go test -v `go list ./... | grep -v vendor/` -gocheck.v=true
 
 coveralls: coverage.out
-	$(GOM) exec $(BINPATH)/goveralls -service travis-ci.org -coverprofile=coverage.out -repotoken=$(COVERALLS_TOKEN)
+	$(BINPATH)/goveralls -service travis-ci.org -coverprofile=coverage.out -repotoken=$(COVERALLS_TOKEN)
 
 mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-package:
-	rm -rf root/
-	mkdir -p root/usr/bin/ root/usr/share/man/man1/ root/etc/bash_completion.d
-	cp $(BINPATH)/aptly root/usr/bin
-	cp man/aptly.1 root/usr/share/man/man1
-	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
-	gzip root/usr/share/man/man1/aptly.1
-	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
-	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 --deb-recommends graphviz -C root/ .
-	mv aptly_$(VERSION)_*.deb ~
-
 src-package:
 	rm -rf aptly-$(VERSION)
 	mkdir -p aptly-$(VERSION)/src/github.com/smira/aptly/
 	cd aptly-$(VERSION)/src/github.com/smira/ && git clone https://github.com/smira/aptly && cd aptly && git checkout v$(VERSION)
-	cd aptly-$(VERSION)/src/github.com/smira/aptly && gom -production install
-	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . \( -name .git -o -name .bzr -o -name .hg \) -print | xargs rm -rf
-	rm -rf aptly-$(VERSION)/src/github.com/smira/aptly/_vendor/{pkg,bin}
 	mkdir -p aptly-$(VERSION)/bash_completion.d
 	(cd aptly-$(VERSION)/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/$(VERSION)/aptly)
 	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
 	rm -rf aptly-$(VERSION)
-	curl -T aptly-$(VERSION)-src.tar.bz2 -usmira:$(BINTRAY_KEY) https://api.bintray.com/content/smira/aptly/aptly/$(VERSION)/$(VERSION)/aptly-$(VERSION)-src.tar.bz2
 
-.PHONY: coverage.out
+goxc:
+	rm -rf root/
+	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d
+	cp man/aptly.1 root/usr/share/man/man1
+	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
+	gzip root/usr/share/man/man1/aptly.1
+	goxc -pv=$(VERSION) -max-processors=4 $(GOXC_OPTS)
+
+man:
+	make -C man
+
+version:
+	@echo $(VERSION)
+
+.PHONY: coverage.out man version

README.rst

@@ -11,8 +11,8 @@ aptly
 .. image:: https://badges.gitter.im/Join Chat.svg
     :target: https://gitter.im/smira/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
 
-.. image:: http://goreportcard.com/badge/gojp/goreportcard
-    :target: http://goreportcard.com/report/gojp/goreportcard
+.. image:: http://goreportcard.com/badge/smira/aptly
+    :target: http://goreportcard.com/report/smira/aptly
 
 Aptly is a swiss army knife for Debian repository management.
 
@@ -48,7 +48,7 @@ To install aptly on Debian/Ubuntu, add new repository to /etc/apt/sources.list::
 
 And import key that is used to sign the release::
 
-    $ apt-key adv --keyserver keys.gnupg.net --recv-keys E083A3782A194991
+    $ apt-key adv --keyserver keys.gnupg.net --recv-keys 9E3E53F19C7DE460
 
 After that you can install aptly as any other software package::
 
@@ -64,18 +64,46 @@ If you would like to use nightly builds (unstable), please use following reposit
 
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.3+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.6+ required)::
 
-    go get -u github.com/mattn/gom
     mkdir -p $GOPATH/src/github.com/smira/aptly
     git clone https://github.com/smira/aptly $GOPATH/src/github.com/smira/aptly
     cd $GOPATH/src/github.com/smira/aptly
-    gom -production install
-    gom build -o $GOPATH/bin/aptly
+    make install
 
-Aptly is using `gom <https://github.com/mattn/gom>`_ to fix external dependencies, so regular ``go get github.com/smira/aptly``
-should work as well, but might fail or produce different result (if external libraries got updated).
+Binary would be installed to ```$GOPATH/bin/aptly``.
 
-If you don't have Go installed (or older version), you can easily install Go using `gvm <https://github.com/moovweb/gvm/>`_.
+Integrations
+------------
 
+Vagrant:
 
+-   `Vagrant configuration <https://github.com/sepulworld/aptly-vagrant>`_ by
+    Zane Williamson, allowing to bring two virtual servers, one with aptly installed
+    and another one set up to install packages from repository published by aptly
+
+Docker:
+
+-    `Docker container <https://github.com/mikepurvis/aptly-docker>`_ with aptly inside by Mike Purvis
+-    `Docker container <https://github.com/bryanhong/docker-aptly>`_ with aptly and nginx by Bryan Hong
+
+With configuration management systems:
+
+-   `Chef cookbook <https://github.com/hw-cookbooks/aptly>`_ by Aaron Baer
+    (Heavy Water Operations, LLC)
+-   `Puppet module <https://github.com/alphagov/puppet-aptly>`_ by
+    Government Digital Services
+-   `Puppet module <https://github.com/tubemogul/puppet-aptly>`_ by
+    TubeMogul
+-   `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by
+    Forrest Alvarez and Brian Jackson
+-   `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine
+
+CLI for aptly API:
+
+-   `Ruby aptly CLI/library <https://github.com/sepulworld/aptly_cli>`_ by Zane Williamson
+-   `Python aptly CLI (good for CI) <https://github.com/TimSusa/aptly_api_cli>`_ by Tim Susa
+
+Scala sbt:
+
+-   `sbt aptly plugin <https://github.com/amalakar/sbt-aptly>`_ by Arup Malakar

_man/gen.go

@@ -2,16 +2,17 @@ package main
 
 import (
 	"fmt"
-	"github.com/smira/aptly/cmd"
-	"github.com/smira/commander"
-	"github.com/smira/flag"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"text/template"
+
+	"github.com/smira/aptly/cmd"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func allFlags(flags *flag.FlagSet) []*flag.Flag {
@@ -43,22 +44,42 @@ func capitalize(s string) string {
 	return strings.Join(parts, " ")
 }
 
+var authorsS string
+
+func authors() string {
+	return authorsS
+}
+
 func main() {
 	command := cmd.RootCommand()
 	command.UsageLine = "aptly"
 	command.Dispatch(nil)
 
-	_, _File, _, _ := runtime.Caller(0)
-	_File, _ = filepath.Abs(_File)
+	_File, _ := filepath.Abs("./man")
 
 	templ := template.New("man").Funcs(template.FuncMap{
 		"allFlags":    allFlags,
 		"findCommand": findCommand,
 		"toUpper":     strings.ToUpper,
 		"capitalize":  capitalize,
+		"authors":     authors,
 	})
 	template.Must(templ.ParseFiles(filepath.Join(filepath.Dir(_File), "aptly.1.ronn.tmpl")))
 
+	authorsF, err := os.Open(filepath.Join(filepath.Dir(_File), "..", "AUTHORS"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsB, err := ioutil.ReadAll(authorsF)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsF.Close()
+
+	authorsS = string(authorsB)
+
 	output, err := os.Create(filepath.Join(filepath.Dir(_File), "aptly.1.ronn"))
 	if err != nil {
 		log.Fatal(err)

api/api.go

@@ -3,12 +3,13 @@ package api
 
 import (
 	"fmt"
+	"sort"
+	"time"
+
 	"github.com/gin-gonic/gin"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
-	"sort"
-	"time"
 )
 
 // Lock order acquisition (canonical):
@@ -22,35 +23,80 @@ func apiVersion(c *gin.Context) {
 	c.JSON(200, gin.H{"Version": aptly.Version})
 }
 
-// Periodically flushes CollectionFactory to free up memory used by collections,
-// flushing caches.
+const (
+	acquiredb = iota
+	releasedb
+)
+
+// Flushes all collections which cache in-memory objects
+func flushColections() {
+	// lock everything to eliminate in-progress calls
+	r := context.CollectionFactory().RemoteRepoCollection()
+	r.Lock()
+	defer r.Unlock()
+
+	l := context.CollectionFactory().LocalRepoCollection()
+	l.Lock()
+	defer l.Unlock()
+
+	s := context.CollectionFactory().SnapshotCollection()
+	s.Lock()
+	defer s.Unlock()
+
+	p := context.CollectionFactory().PublishedRepoCollection()
+	p.Lock()
+	defer p.Unlock()
+
+	// all collections locked, flush them
+	context.CollectionFactory().Flush()
+}
+
+// Periodically flushes CollectionFactory to free up memory used by
+// collections, flushing caches. If the two channels are provided,
+// they are used to acquire and release the database.
 //
 // Should be run in goroutine!
-func cacheFlusher() {
+func cacheFlusher(requests chan int, acks chan error) {
 	ticker := time.Tick(15 * time.Minute)
 
 	for {
 		<-ticker
 
-		// lock everything to eliminate in-progress calls
-		r := context.CollectionFactory().RemoteRepoCollection()
-		r.Lock()
-		defer r.Unlock()
+		// if aptly API runs in -no-lock mode,
+		// caches are flushed when DB is closed anyway, no need
+		// to flush them here
+		if requests == nil {
+			flushColections()
+		}
+	}
+}
 
-		l := context.CollectionFactory().LocalRepoCollection()
-		l.Lock()
-		defer l.Unlock()
-
-		s := context.CollectionFactory().SnapshotCollection()
-		s.Lock()
-		defer s.Unlock()
-
-		p := context.CollectionFactory().PublishedRepoCollection()
-		p.Lock()
-		defer p.Unlock()
-
-		// all collections locked, flush them
-		context.CollectionFactory().Flush()
+// Acquire database lock and release it when not needed anymore. Two
+// channels must be provided. The first one is to receive requests to
+// acquire/release the database and the second one is to send acks.
+//
+// Should be run in a goroutine!
+func acquireDatabase(requests chan int, acks chan error) {
+	clients := 0
+	for {
+		request := <-requests
+		switch request {
+		case acquiredb:
+			if clients == 0 {
+				acks <- context.ReOpenDatabase()
+			} else {
+				acks <- nil
+			}
+			clients++
+		case releasedb:
+			clients--
+			if clients == 0 {
+				flushColections()
+				acks <- context.CloseDatabase()
+			} else {
+				acks <- nil
+			}
+		}
 	}
 }
 
@@ -97,6 +143,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 			nil, context.DependencyOptions(), architecturesList)
 		if err != nil {
 			c.Fail(500, fmt.Errorf("unable to search: %s", err))
+			return
 		}
 	}
 

api/files.go

@@ -2,11 +2,12 @@ package api
 
 import (
 	"fmt"
-	"github.com/gin-gonic/gin"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/gin-gonic/gin"
 )
 
 func verifyPath(path string) bool {

api/graph.go

@@ -3,15 +3,16 @@ package api
 import (
 	"bytes"
 	"fmt"
-	"github.com/gin-gonic/gin"
-	"github.com/smira/aptly/deb"
 	"io"
 	"mime"
 	"os"
 	"os/exec"
+
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/deb"
 )
 
-// GET /api/graph.:ext
+// GET /api/graph.:ext?layout=[vertical|horizontal(default)]
 func apiGraph(c *gin.Context) {
 	var (
 		err    error
@@ -19,6 +20,7 @@ func apiGraph(c *gin.Context) {
 	)
 
 	ext := c.Params.ByName("ext")
+	layout := c.Request.URL.Query().Get("layout")
 
 	factory := context.CollectionFactory()
 
@@ -31,7 +33,7 @@ func apiGraph(c *gin.Context) {
 	factory.PublishedRepoCollection().RLock()
 	defer factory.PublishedRepoCollection().RUnlock()
 
-	graph, err := deb.BuildGraph(factory)
+	graph, err := deb.BuildGraph(factory, layout)
 	if err != nil {
 		c.JSON(500, err)
 		return
@@ -39,6 +41,13 @@ func apiGraph(c *gin.Context) {
 
 	buf := bytes.NewBufferString(graph.String())
 
+	if ext == "dot" || ext == "gv" {
+		// If the raw dot data is requested, return it as string.
+		// This allows client-side rendering rather than server-side.
+		c.String(200, buf.String())
+		return
+	}
+
 	command := exec.Command("dot", "-T"+ext)
 	command.Stderr = os.Stderr
 

api/publish.go

@@ -2,10 +2,11 @@ package api
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/gin-gonic/gin"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
-	"strings"
 )
 
 // SigningOptions is a shared between publish API GPG options structure
@@ -97,6 +98,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		Label          string
 		Origin         string
 		ForceOverwrite bool
+		SkipContents   *bool
 		Architectures  []string
 		Signing        SigningOptions
 	}
@@ -183,6 +185,11 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 	published.Origin = b.Origin
 	published.Label = b.Label
 
+	published.SkipContents = context.Config().SkipContentsPublishing
+	if b.SkipContents != nil {
+		published.SkipContents = *b.SkipContents
+	}
+
 	duplicate := collection.CheckDuplicate(published)
 	if duplicate != nil {
 		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
@@ -199,6 +206,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 	err = collection.Add(published)
 	if err != nil {
 		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		return
 	}
 
 	c.JSON(201, published)
@@ -213,6 +221,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 	var b struct {
 		ForceOverwrite bool
 		Signing        SigningOptions
+		SkipContents   *bool
 		Snapshots      []struct {
 			Component string `binding:"required"`
 			Name      string `binding:"required"`
@@ -289,22 +298,30 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		}
 	} else {
 		c.Fail(500, fmt.Errorf("unknown published repository type"))
+		return
+	}
+
+	if b.SkipContents != nil {
+		published.SkipContents = *b.SkipContents
 	}
 
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
 	if err != nil {
 		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		return
 	}
 
 	err = collection.Update(published)
 	if err != nil {
 		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		return
 	}
 
 	err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
 		context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
 	if err != nil {
 		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		return
 	}
 
 	c.JSON(200, published)

api/repos.go

@@ -2,13 +2,14 @@ package api
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
+
 	"github.com/gin-gonic/gin"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
-	"os"
-	"path/filepath"
 )
 
 // GET /api/repos
@@ -60,9 +61,9 @@ func apiReposCreate(c *gin.Context) {
 // PUT /api/repos/:name
 func apiReposEdit(c *gin.Context) {
 	var b struct {
-		Comment             string
-		DefaultDistribution string
-		DefaultComponent    string
+		Comment             *string
+		DefaultDistribution *string
+		DefaultComponent    *string
 	}
 
 	if !c.Bind(&b) {
@@ -79,14 +80,14 @@ func apiReposEdit(c *gin.Context) {
 		return
 	}
 
-	if b.Comment != "" {
-		repo.Comment = b.Comment
+	if b.Comment != nil {
+		repo.Comment = *b.Comment
 	}
-	if b.DefaultDistribution != "" {
-		repo.DefaultDistribution = b.DefaultDistribution
+	if b.DefaultDistribution != nil {
+		repo.DefaultDistribution = *b.DefaultDistribution
 	}
-	if b.DefaultComponent != "" {
-		repo.DefaultComponent = b.DefaultComponent
+	if b.DefaultComponent != nil {
+		repo.DefaultComponent = *b.DefaultComponent
 	}
 
 	err = collection.Update(repo)
@@ -315,12 +316,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
 	}
 
-	packageFiles, failedFiles, err = deb.CollectPackageFiles(sources, reporter)
-
-	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to collect package files: %s", err))
-		return
-	}
+	packageFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
 
 	list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
 	if err != nil {
@@ -329,7 +325,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 	}
 
 	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), reporter)
+		context.CollectionFactory().PackageCollection(), reporter, nil)
 	failedFiles = append(failedFiles, failedFiles2...)
 
 	if err != nil {

api/router.go

@@ -1,9 +1,10 @@
 package api
 
 import (
+	"net/http"
+
 	"github.com/gin-gonic/gin"
 	ctx "github.com/smira/aptly/context"
-	"net/http"
 )
 
 var context *ctx.AptlyContext
@@ -12,11 +13,41 @@ var context *ctx.AptlyContext
 func Router(c *ctx.AptlyContext) http.Handler {
 	context = c
 
-	go cacheFlusher()
-
 	router := gin.Default()
 	router.Use(gin.ErrorLogger())
 
+	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
+		// We use a goroutine to count the number of
+		// concurrent requests. When no more requests are
+		// running, we close the database to free the lock.
+		requests := make(chan int)
+		acks := make(chan error)
+
+		go acquireDatabase(requests, acks)
+		go cacheFlusher(requests, acks)
+
+		router.Use(func(c *gin.Context) {
+			requests <- acquiredb
+			err := <-acks
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+			defer func() {
+				requests <- releasedb
+				err = <-acks
+				if err != nil {
+					c.Fail(500, err)
+					return
+				}
+			}()
+			c.Next()
+		})
+
+	} else {
+		go cacheFlusher(nil, nil)
+	}
+
 	root := router.Group("/api")
 
 	{

api/snapshot.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"fmt"
+
 	"github.com/gin-gonic/gin"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/deb"

aptly/interfaces.go

@@ -3,8 +3,9 @@
 package aptly
 
 import (
-	"github.com/smira/aptly/utils"
 	"io"
+
+	"github.com/smira/aptly/utils"
 )
 
 // PackagePool is asbtraction of package pool storage.
@@ -82,7 +83,7 @@ type Downloader interface {
 	// Download starts new download task
 	Download(url string, destination string, result chan<- error)
 	// DownloadWithChecksum starts new download task with checksum verification
-	DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool)
+	DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int)
 	// Pause pauses task processing
 	Pause()
 	// Resume resumes task processing

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
-// Version of aptly
-const Version = "0.9.5"
+// Version of aptly (filled in at link time)
+var Version string
 
-// Enable debugging features?
+// EnableDebug triggers some debugging features
 const EnableDebug = false

cmd/api_serve.go

@@ -2,10 +2,16 @@ package cmd
 
 import (
 	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+
 	"github.com/smira/aptly/api"
+	"github.com/smira/aptly/systemd/activation"
+	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"net/http"
 )
 
 func aptlyAPIServe(cmd *commander.Command, args []string) error {
@@ -18,10 +24,53 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	listen := context.Flags().Lookup("listen").Value.String()
+	// There are only two working options for aptly's rootDir:
+	//   1. rootDir does not exist, then we'll create it
+	//   2. rootDir exists and is writable
+	// anything else must fail.
+	// E.g.: Running the service under a different user may lead to a rootDir
+	// that exists but is not usable due to access permissions.
+	err = utils.DirIsAccessible(context.Config().RootDir)
+	if err != nil {
+		return err
+	}
 
+	// Try to recycle systemd fds for listening
+	listeners, err := activation.Listeners(true)
+	if len(listeners) > 1 {
+		panic("Got more than 1 listener from systemd. This is currently not supported!")
+	}
+	if err == nil && len(listeners) == 1 {
+		listener := listeners[0]
+		defer listener.Close()
+		fmt.Printf("\nTaking over web server at: %s (press Ctrl+C to quit)...\n", listener.Addr().String())
+		err = http.Serve(listener, api.Router(context))
+		if err != nil {
+			return fmt.Errorf("unable to serve: %s", err)
+		}
+		return nil
+	}
+
+	// If there are none: use the listen argument.
+	listen := context.Flags().Lookup("listen").Value.String()
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
 
+	listenURL, err := url.Parse(listen)
+	if err == nil && listenURL.Scheme == "unix" {
+		file := listenURL.Path
+		os.Remove(file)
+		listener, err := net.Listen("unix", file)
+		if err != nil {
+			return fmt.Errorf("failed to listen on: %s\n%s", file, err)
+		}
+		defer listener.Close()
+		err = http.Serve(listener, api.Router(context))
+		if err != nil {
+			return fmt.Errorf("unable to serve: %s", err)
+		}
+		return nil
+	}
+
 	err = http.ListenAndServe(listen, api.Router(context))
 	if err != nil {
 		return fmt.Errorf("unable to serve: %s", err)
@@ -36,16 +85,21 @@ func makeCmdAPIServe() *commander.Command {
 		UsageLine: "serve",
 		Short:     "start API HTTP service",
 		Long: `
-Stat HTTP server with aptly REST API.
+Start HTTP server with aptly REST API. The server can listen to either a port
+or Unix domain socket. When using a socket, Aptly will fully manage the socket
+file. This command also supports taking over from a systemd file descriptors to
+enable systemd socket activation.
 
 Example:
 
   $ aptly api serve -listen=:8080
+  $ aptly api serve -listen=unix:///tmp/aptly.sock
 `,
 		Flag: *flag.NewFlagSet("aptly-serve", flag.ExitOnError),
 	}
 
-	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening")
+	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening or unix://path to listen on a Unix domain socket")
+	cmd.Flag.Bool("no-lock", false, "don't lock the database")
 
 	return cmd
 

cmd/cmd.go

@@ -2,13 +2,16 @@
 package cmd
 
 import (
+	"bytes"
 	"fmt"
+	"os"
+	"text/template"
+	"time"
+
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"time"
 )
 
 // ListPackagesRefList shows list of packages in PackageRefList
@@ -34,6 +37,32 @@ func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 	return
 }
 
+// PrintPackageList shows package list with specified format or default representation
+func PrintPackageList(result *deb.PackageList, format string) error {
+	if format == "" {
+		return result.ForEach(func(p *deb.Package) error {
+			context.Progress().Printf("%s\n", p)
+			return nil
+		})
+	}
+
+	formatTemplate, err := template.New("format").Parse(format)
+	if err != nil {
+		return fmt.Errorf("error parsing -format template: %s", err)
+	}
+
+	return result.ForEach(func(p *deb.Package) error {
+		b := &bytes.Buffer{}
+		err = formatTemplate.Execute(b, p.ExtendedStanza())
+		if err != nil {
+			return fmt.Errorf("error applying template: %s", err)
+		}
+		context.Progress().Printf("%s\n", b.String())
+		return nil
+	})
+
+}
+
 // LookupOption checks boolean flag with default (usually config) and command-line
 // setting
 func LookupOption(defaultValue bool, flags *flag.FlagSet, name string) (result bool) {
@@ -83,7 +112,7 @@ package environment to new version.`,
 	cmd.Flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests")
 	cmd.Flag.Bool("dep-follow-source", false, "when processing dependencies, follow from binary to Source packages")
 	cmd.Flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends")
-	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if depdency is 'a|b'")
+	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
 	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
 

cmd/config_show.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+
 	"github.com/smira/commander"
 )
 

cmd/db_cleanup.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
-	"sort"
-	"strings"
 )
 
 // aptly db cleanup
@@ -153,7 +154,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
 	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
 
-	toDelete := allPackageRefs.Substract(existingPackageRefs)
+	toDelete := allPackageRefs.Subtract(existingPackageRefs)
 
 	// delete packages that are no longer referenced
 	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())

cmd/graph.go

@@ -3,12 +3,18 @@ package cmd
 import (
 	"bytes"
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
 )
 
 func aptlyGraph(cmd *commander.Command, args []string) error {
@@ -19,8 +25,11 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
+	layout := context.Flags().Lookup("layout").Value.String()
+
 	fmt.Printf("Generating graph...\n")
-	graph, err := deb.BuildGraph(context.CollectionFactory())
+	graph, err := deb.BuildGraph(context.CollectionFactory(), layout)
+
 	if err != nil {
 		return err
 	}
@@ -34,9 +43,16 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 	tempfile.Close()
 	os.Remove(tempfile.Name())
 
-	tempfilename := tempfile.Name() + ".png"
+	format := context.Flags().Lookup("format").Value.String()
+	output := context.Flags().Lookup("output").Value.String()
 
-	command := exec.Command("dot", "-Tpng", "-o"+tempfilename)
+	if filepath.Ext(output) != "" {
+		format = filepath.Ext(output)[1:]
+	}
+
+	tempfilename := tempfile.Name() + "." + format
+
+	command := exec.Command("dot", "-T"+format, "-o"+tempfilename)
 	command.Stderr = os.Stderr
 
 	stdin, err := command.StdinPipe()
@@ -64,15 +80,51 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	err = exec.Command("open", tempfilename).Run()
-	if err != nil {
-		fmt.Printf("Rendered to PNG file: %s\n", tempfilename)
-		err = nil
+	defer func() {
+		_ = os.Remove(tempfilename)
+	}()
+
+	if output != "" {
+		err = utils.CopyFile(tempfilename, output)
+		if err != nil {
+			return fmt.Errorf("unable to copy %s -> %s: %s", tempfilename, output, err)
+		}
+
+		fmt.Printf("Output saved to %s\n", output)
+	} else {
+		command := getOpenCommand()
+		fmt.Printf("Rendered to %s file: %s, trying to open it with: %s %s...\n", format, tempfilename, command, tempfilename)
+
+		args := strings.Split(command, " ")
+
+		viewer := exec.Command(args[0], append(args[1:], tempfilename)...)
+		viewer.Stderr = os.Stderr
+		if err = viewer.Start(); err == nil {
+			// Wait for a second so that the visualizer has a chance to
+			// open the input file. This needs to be done even if we're
+			// waiting for the visualizer as it can be just a wrapper that
+			// spawns a browser tab and returns right away.
+			defer func(t <-chan time.Time) {
+				<-t
+			}(time.After(time.Second))
+		}
 	}
 
 	return err
 }
 
+// getOpenCommand tries to guess command to open image for OS
+func getOpenCommand() string {
+	switch runtime.GOOS {
+	case "darwin":
+		return "/usr/bin/open"
+	case "windows":
+		return "cmd /c start"
+	default:
+		return "xdg-open"
+	}
+}
+
 func makeCmdGraph() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyGraph,
@@ -89,5 +141,9 @@ Example:
 `,
 	}
 
+	cmd.Flag.String("format", "png", "render graph to specified format (png, svg, pdf, etc.)")
+	cmd.Flag.String("output", "", "specify output filename, default is to open result in viewer")
+	cmd.Flag.String("layout", "horizontal", "create a more 'vertical' or a more 'horizontal' graph layout")
+
 	return cmd
 }

cmd/mirror.go

@@ -1,10 +1,11 @@
 package cmd
 
 import (
+	"strings"
+
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func getVerifier(flags *flag.FlagSet) (utils.Verifier, error) {

cmd/mirror_create.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
@@ -43,6 +44,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	repo.Filter = context.Flags().Lookup("filter").Value.String()
 	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
 	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
+	repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
 
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
@@ -97,6 +99,7 @@ Example:
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
+	cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/mirror_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/mirror_edit.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"

cmd/mirror_list.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
-	"sort"
 )
 
 func aptlyMirrorList(cmd *commander.Command, args []string) error {

cmd/mirror_rename.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )

cmd/mirror_search.go

@@ -8,11 +8,13 @@ import (
 func makeCmdMirrorSearch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotMirrorRepoSearch,
-		UsageLine: "search <name> <package-query>",
+		UsageLine: "search <name> [<package-query>]",
 		Short:     "search mirror for packages matching query",
 		Long: `
 Command search displays list of packages in mirror that match package query
 
+If query is not specified, all the packages are displayed.
+
 Example:
 
     $ aptly mirror search wheezy-main '$Architecture (i386), Name (% *-dev)'
@@ -21,6 +23,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/mirror_show.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyMirrorShow(cmd *commander.Command, args []string) error {

cmd/mirror_update.go

@@ -2,14 +2,15 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+	"os/signal"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"os/signal"
-	"strings"
 )
 
 func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
@@ -40,6 +41,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
+	maxTries := context.Flags().Lookup("max-tries").Value.Get().(int)
 
 	verifier, err := getVerifier(context.Flags())
 	if err != nil {
@@ -52,7 +54,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	context.Progress().Printf("Downloading & parsing package files...\n")
-	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch)
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch, maxTries)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -87,7 +89,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 
 	defer func() {
 		// on any interruption, unlock the mirror
-		err := context.ReOpenDatabase()
+		err = context.ReOpenDatabase()
 		if err == nil {
 			repo.MarkAsIdle()
 			context.CollectionFactory().RemoteRepoCollection().Update(repo)
@@ -121,7 +123,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	// In separate goroutine (to avoid blocking main), push queue to downloader
 	go func() {
 		for _, task := range queue {
-			context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
+			context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch, maxTries)
 		}
 
 		// We don't need queue after this point
@@ -129,7 +131,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}()
 
 	// Wait for all downloads to finish
-	errors := make([]string, 0)
+	var errors []string
 
 	for count > 0 {
 		select {
@@ -148,7 +150,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	signal.Stop(sigch)
 
 	if len(errors) > 0 {
-		return fmt.Errorf("unable to update: download errors:\n  %s\n", strings.Join(errors, "\n  "))
+		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
 	}
 
 	err = context.ReOpenDatabase()
@@ -187,6 +189,7 @@ Example:
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
+	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/package_search.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
@@ -9,15 +10,23 @@ import (
 )
 
 func aptlyPackageSearch(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) != 1 {
+	var (
+		err error
+		q   deb.PackageQuery
+	)
+
+	if len(args) > 1 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
-	q, err := query.Parse(args[0])
-	if err != nil {
-		return fmt.Errorf("unable to search: %s", err)
+	if len(args) == 1 {
+		q, err = query.Parse(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+	} else {
+		q = &deb.MatchAllQuery{}
 	}
 
 	result := q.Query(context.CollectionFactory().PackageCollection())
@@ -25,10 +34,8 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("no results")
 	}
 
-	result.ForEach(func(p *deb.Package) error {
-		context.Progress().Printf("%s\n", p)
-		return nil
-	})
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format)
 
 	return err
 }
@@ -36,10 +43,12 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 func makeCmdPackageSearch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPackageSearch,
-		UsageLine: "search <package-query>",
+		UsageLine: "search [<package-query>]",
 		Short:     "search for packages matching query",
 		Long: `
-Command search displays list of packages in whole DB that match package query
+Command search displays list of packages in whole DB that match package query.
+
+If query is not specified, all the packages are displayed.
 
 Example:
 
@@ -48,5 +57,7 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-package-search", flag.ExitOnError),
 	}
 
+	cmd.Flag.String("format", "", "custom format for result printing")
+
 	return cmd
 }

cmd/package_show.go

@@ -3,11 +3,12 @@ package cmd
 import (
 	"bufio"
 	"fmt"
+	"os"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
 )
 
 func printReferencesTo(p *deb.Package) (err error) {
@@ -125,7 +126,7 @@ inclusion into mirrors/snapshots/local repos is shown.
 
 Example:
 
-    $ aptly package show nginx-light_1.2.1-2.2+wheezy2_i386'
+    $ aptly package show 'nginx-light_1.2.1-2.2+wheezy2_i386'
 `,
 		Flag: *flag.NewFlagSet("aptly-package-show", flag.ExitOnError),
 	}

cmd/publish.go

@@ -37,6 +37,7 @@ func makeCmdPublish() *commander.Command {
 			makeCmdPublishSnapshot(),
 			makeCmdPublishSwitch(),
 			makeCmdPublishUpdate(),
+			makeCmdPublishShow(),
 		},
 	}
 }

cmd/publish_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )

cmd/publish_list.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
-	"sort"
 )
 
 func aptlyPublishList(cmd *commander.Command, args []string) error {

cmd/publish_repo.go

@@ -41,6 +41,7 @@ Example:
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")

cmd/publish_show.go

@@ -0,0 +1,81 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyPublishShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	repo, err := context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	if repo.Storage != "" {
+		fmt.Printf("Storage: %s\n", repo.Storage)
+	}
+	fmt.Printf("Prefix: %s\n", repo.Prefix)
+	if repo.Distribution != "" {
+		fmt.Printf("Distribution: %s\n", repo.Distribution)
+	}
+	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, " "))
+
+	fmt.Printf("Sources:\n")
+	for component, sourceID := range repo.Sources {
+		var name string
+		if repo.SourceKind == "snapshot" {
+			source, err := context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
+			if err != nil {
+				continue
+			}
+			name = source.Name
+		} else if repo.SourceKind == "local" {
+			source, err := context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+			if err != nil {
+				continue
+			}
+			name = source.Name
+		}
+
+		if name != "" {
+			fmt.Printf("  %s: %s [%s]\n", component, name, repo.SourceKind)
+		}
+	}
+
+	return err
+}
+
+func makeCmdPublishShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishShow,
+		UsageLine: "show <distribution> [[<endpoint>:]<prefix>]",
+		Short:     "shows details of published repository",
+		Long: `
+Command show displays full information of a published repository.
+
+Example:
+
+    $ aptly publish show wheezy
+`,
+	}
+
+	return cmd
+}

cmd/publish_snapshot.go

@@ -2,12 +2,13 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
@@ -116,8 +117,14 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
-	published.Origin = cmd.Flag.Lookup("origin").Value.String()
-	published.Label = cmd.Flag.Lookup("label").Value.String()
+	published.Origin = context.Flags().Lookup("origin").Value.String()
+	published.Label = context.Flags().Lookup("label").Value.String()
+
+	published.SkipContents = context.Config().SkipContentsPublishing
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
 
 	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
@@ -203,6 +210,7 @@ Example:
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")

cmd/publish_switch.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
@@ -90,6 +91,10 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 			"the same package pool.\n")
 	}
 
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
@@ -143,6 +148,7 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 

cmd/publish_update.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -54,6 +55,10 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 			"the same package pool.\n")
 	}
 
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
@@ -102,6 +107,7 @@ Example:
 	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 
 	return cmd

cmd/repo.go

@@ -21,6 +21,7 @@ func makeCmdRepo() *commander.Command {
 			makeCmdRepoShow(),
 			makeCmdRepoRename(),
 			makeCmdRepoSearch(),
+			makeCmdRepoInclude(),
 		},
 	}
 }

cmd/repo_add.go

@@ -2,12 +2,13 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
 )
 
 func aptlyRepoAdd(cmd *commander.Command, args []string) error {
@@ -42,15 +43,12 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	var packageFiles, failedFiles []string
 
-	packageFiles, failedFiles, err = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
-	if err != nil {
-		return fmt.Errorf("unable to collect package files: %s", err)
-	}
+	packageFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
 
 	var processedFiles, failedFiles2 []string
 
 	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()})
+		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil)
 	failedFiles = append(failedFiles, failedFiles2...)
 	if err != nil {
 		return fmt.Errorf("unable to import package files: %s", err)

cmd/repo_create.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -9,7 +10,7 @@ import (
 
 func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	var err error
-	if len(args) != 1 {
+	if !(len(args) == 1 || (len(args) == 4 && args[1] == "from" && args[2] == "snapshot")) {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
@@ -18,6 +19,30 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
 	repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
 
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		repo.Uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(args) == 4 {
+		var snapshot *deb.Snapshot
+
+		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(args[3])
+		if err != nil {
+			return fmt.Errorf("unable to load source snapshot: %s", err)
+		}
+
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to load source snapshot: %s", err)
+		}
+
+		repo.UpdateRefList(snapshot.RefList())
+	}
+
 	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add local repo: %s", err)
@@ -30,16 +55,21 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 func makeCmdRepoCreate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoCreate,
-		UsageLine: "create <name>",
+		UsageLine: "create <name> [ from snapshot <snapshot> ]",
 		Short:     "create local repository",
 		Long: `
 Create local package repository. Repository would be empty when
 created, packages could be added from files, copied or moved from
 another local repository or imported from the mirror.
 
+If local package repository is created from snapshot, repo initial
+contents are copied from snapsot contents.
+
 Example:
 
   $ aptly repo create testing
+
+  $ aptly repo create mysql35 from snapshot mysql-35-2017
 `,
 		Flag: *flag.NewFlagSet("aptly-repo-create", flag.ExitOnError),
 	}
@@ -47,6 +77,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "main", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/repo_edit.go

@@ -2,6 +2,9 @@ package cmd
 
 import (
 	"fmt"
+
+	"github.com/AlekSi/pointer"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -23,16 +26,30 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	if context.Flags().Lookup("comment").Value.String() != "" {
-		repo.Comment = context.Flags().Lookup("comment").Value.String()
-	}
+	var uploadersFile *string
 
-	if context.Flags().Lookup("distribution").Value.String() != "" {
-		repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
-	}
+	context.Flags().Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "comment":
+			repo.Comment = flag.Value.String()
+		case "distribution":
+			repo.DefaultDistribution = flag.Value.String()
+		case "component":
+			repo.DefaultComponent = flag.Value.String()
+		case "uploaders-file":
+			uploadersFile = pointer.ToString(flag.Value.String())
+		}
+	})
 
-	if context.Flags().Lookup("component").Value.String() != "" {
-		repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
+	if uploadersFile != nil {
+		if *uploadersFile != "" {
+			repo.Uploaders, err = deb.NewUploadersFromFile(*uploadersFile)
+			if err != nil {
+				return err
+			}
+		} else {
+			repo.Uploaders = nil
+		}
 	}
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
@@ -63,6 +80,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_include.go

@@ -0,0 +1,235 @@
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"text/template"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoInclude(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	verifier, err := getVerifier(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+	}
+
+	if verifier == nil {
+		verifier = &utils.GpgVerifier{}
+	}
+
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
+	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
+	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
+
+	repoTemplate, err := template.New("repo").Parse(context.Flags().Lookup("repo").Value.Get().(string))
+	if err != nil {
+		return fmt.Errorf("error parsing -repo template: %s", err)
+	}
+
+	uploaders := (*deb.Uploaders)(nil)
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+
+		for i := range uploaders.Rules {
+			uploaders.Rules[i].CompiledCondition, err = query.Parse(uploaders.Rules[i].Condition)
+			if err != nil {
+				return fmt.Errorf("error parsing query %s: %s", uploaders.Rules[i].Condition, err)
+			}
+		}
+	}
+
+	reporter := &aptly.ConsoleResultReporter{Progress: context.Progress()}
+
+	var changesFiles, failedFiles, processedFiles []string
+
+	changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)
+
+	for _, path := range changesFiles {
+		var changes *deb.Changes
+
+		changes, err = deb.NewChanges(path)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", path, err)
+			continue
+		}
+
+		err = changes.VerifyAndParse(acceptUnsigned, ignoreSignatures, verifier)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		err = changes.Prepare()
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		repoName := &bytes.Buffer{}
+		err = repoTemplate.Execute(repoName, changes.Stanza)
+		if err != nil {
+			return fmt.Errorf("error applying template to repo: %s", err)
+		}
+
+		context.Progress().Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
+
+		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(repoName.String())
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		currentUploaders := uploaders
+		if repo.Uploaders != nil {
+			currentUploaders = repo.Uploaders
+			for i := range currentUploaders.Rules {
+				currentUploaders.Rules[i].CompiledCondition, err = query.Parse(currentUploaders.Rules[i].Condition)
+				if err != nil {
+					return fmt.Errorf("error parsing query %s: %s", currentUploaders.Rules[i].Condition, err)
+				}
+			}
+		}
+
+		if currentUploaders != nil {
+			if err = currentUploaders.IsAllowed(changes); err != nil {
+				failedFiles = append(failedFiles, path)
+				reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
+					changes.ChangesName, changes.SignatureKeys, err)
+				changes.Cleanup()
+				continue
+			}
+		}
+
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to load repo: %s", err)
+		}
+
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to load packages: %s", err)
+		}
+
+		packageFiles, _ := deb.CollectPackageFiles([]string{changes.TempDir}, reporter)
+
+		var restriction deb.PackageQuery
+
+		restriction, err = changes.PackageQuery()
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		var processedFiles2, failedFiles2 []string
+
+		processedFiles2, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+			context.CollectionFactory().PackageCollection(), reporter, restriction)
+
+		if err != nil {
+			return fmt.Errorf("unable to import package files: %s", err)
+		}
+
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+		if err != nil {
+			return fmt.Errorf("unable to save: %s", err)
+		}
+
+		err = changes.Cleanup()
+		if err != nil {
+			return err
+		}
+
+		for _, file := range failedFiles2 {
+			failedFiles = append(failedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		for _, file := range processedFiles2 {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		processedFiles = append(processedFiles, path)
+	}
+
+	if !noRemoveFiles {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err := os.Remove(file)
+			if err != nil {
+				return fmt.Errorf("unable to remove file: %s", err)
+			}
+		}
+	}
+
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
+	return err
+}
+
+func makeCmdRepoInclude() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoInclude,
+		UsageLine: "include <file.changes>|<directory> ...",
+		Short:     "add packages to local repositories based on .changes files",
+		Long: `
+Command include looks for .changes files in list of arguments or specified directories. Each
+.changes file is verified, parsed, referenced files are put into separate temporary directory
+and added into local repository. Successfully imported files are removed by default.
+
+Additionally uploads could be restricted with <uploaders.json> file. Rules in this file control
+uploads based on GPG key ID of .changes file signature and queries on .changes file fields.
+
+Example:
+
+  $ aptly repo include -repo=foo-release incoming/
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-include", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("no-remove-files", false, "don't remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
+	cmd.Flag.String("repo", "{{.Distribution}}", "which repo should files go to, defaults to Distribution field of .changes file")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of .changes file signature")
+	cmd.Flag.Bool("accept-unsigned", false, "accept unsigned .changes files")
+	cmd.Flag.String("uploaders-file", "", "path to uploaders.json file")
+
+	return cmd
+}

cmd/repo_list.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
-	"sort"
 )
 
 func aptlyRepoList(cmd *commander.Command, args []string) error {

cmd/repo_move.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"sort"
 )
 
 func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {

cmd/repo_remove.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"

cmd/repo_rename.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )

cmd/repo_search.go

@@ -8,11 +8,13 @@ import (
 func makeCmdRepoSearch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotMirrorRepoSearch,
-		UsageLine: "search <name> <package-query>",
+		UsageLine: "search <name> [<package-query>]",
 		Short:     "search repo for packages matching query",
 		Long: `
 Command search displays list of packages in local repository that match package query
 
+If query is not specified, all the packages are displayed.
+
 Example:
 
     $ aptly repo search my-software '$Architecture (i386), Name (% *-dev)'
@@ -21,6 +23,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/repo_show.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -29,6 +30,9 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Comment: %s\n", repo.Comment)
 	fmt.Printf("Default Distribution: %s\n", repo.DefaultDistribution)
 	fmt.Printf("Default Component: %s\n", repo.DefaultComponent)
+	if repo.Uploaders != nil {
+		fmt.Printf("Uploaders: %s\n", repo.Uploaders)
+	}
 	fmt.Printf("Number of packages: %d\n", repo.NumPackages())
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)

cmd/run.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+
 	ctx "github.com/smira/aptly/context"
 	"github.com/smira/commander"
 )
@@ -14,7 +16,7 @@ func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode
 			if !ok {
 				panic(r)
 			}
-			fmt.Println("ERROR:", fatal.Message)
+			fmt.Fprintln(os.Stderr, "ERROR:", fatal.Message)
 			returnCode = fatal.ReturnCode
 		}
 	}()

cmd/serve.go

@@ -2,16 +2,17 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
-	"github.com/smira/commander"
-	"github.com/smira/flag"
 	"net"
 	"net/http"
 	"os"
 	"sort"
 	"strings"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyServe(cmd *commander.Command, args []string) error {
@@ -22,6 +23,17 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
+	// There are only two working options for aptly's rootDir:
+	//   1. rootDir does not exist, then we'll create it
+	//   2. rootDir exists and is writable
+	// anything else must fail.
+	// E.g.: Running the service under a different user may lead to a rootDir
+	// that exists but is not usable due to access permissions.
+	err = utils.DirIsAccessible(context.Config().RootDir)
+	if err != nil {
+		return err
+	}
+
 	if context.CollectionFactory().PublishedRepoCollection().Len() == 0 {
 		fmt.Printf("No published repositories, unable to serve.\n")
 		return nil

cmd/snapshot_create.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )

cmd/snapshot_diff.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/snapshot_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/snapshot_filter.go

@@ -2,12 +2,13 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"sort"
-	"strings"
 )
 
 func aptlySnapshotFilter(cmd *commander.Command, args []string) error {

cmd/snapshot_list.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )

cmd/snapshot_merge.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
-	"strings"
 )
 
 func aptlySnapshotMerge(cmd *commander.Command, args []string) error {

cmd/snapshot_pull.go

@@ -2,12 +2,13 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"sort"
-	"strings"
 )
 
 func aptlySnapshotPull(cmd *commander.Command, args []string) error {

cmd/snapshot_rename.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )

cmd/snapshot_search.go

@@ -2,16 +2,21 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"sort"
 )
 
 func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) != 2 {
+	var (
+		err error
+		q   deb.PackageQuery
+	)
+
+	if len(args) < 1 || len(args) > 2 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
@@ -68,9 +73,13 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 
 	list.PrepareIndex()
 
-	q, err := query.Parse(args[1])
-	if err != nil {
-		return fmt.Errorf("unable to search: %s", err)
+	if len(args) == 2 {
+		q, err = query.Parse(args[1])
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+	} else {
+		q = &deb.MatchAllQuery{}
 	}
 
 	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
@@ -100,10 +109,8 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		return fmt.Errorf("no results")
 	}
 
-	result.ForEach(func(p *deb.Package) error {
-		context.Progress().Printf("%s\n", p)
-		return nil
-	})
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format)
 
 	return err
 }
@@ -111,11 +118,13 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 func makeCmdSnapshotSearch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotMirrorRepoSearch,
-		UsageLine: "search <name> <package-query>",
+		UsageLine: "search <name> [<package-query>]",
 		Short:     "search snapshot for packages matching query",
 		Long: `
 Command search displays list of packages in snapshot that match package query
 
+If query is not specified, all the packages are displayed.
+
 Example:
 
     $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'
@@ -124,6 +133,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/snapshot_show.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -29,6 +30,35 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Created At: %s\n", snapshot.CreatedAt.Format("2006-01-02 15:04:05 MST"))
 	fmt.Printf("Description: %s\n", snapshot.Description)
 	fmt.Printf("Number of packages: %d\n", snapshot.NumPackages())
+	if len(snapshot.SourceIDs) > 0 {
+		fmt.Printf("Sources:\n")
+		for _, sourceID := range snapshot.SourceIDs {
+			var name string
+			if snapshot.SourceKind == "snapshot" {
+				source, err := context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			} else if snapshot.SourceKind == "local" {
+				source, err := context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			} else if snapshot.SourceKind == "repo" {
+				source, err := context.CollectionFactory().RemoteRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			}
+
+			if name != "" {
+				fmt.Printf("  %s [%s]\n", name, snapshot.SourceKind)
+			}
+		}
+	}
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {

cmd/snapshot_verify.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
-	"sort"
 )
 
 func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
@@ -31,25 +32,25 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 
 	packageList, err := deb.NewPackageListFromRefList(snapshots[0].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
-		fmt.Errorf("unable to load packages: %s", err)
+		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
 	sourcePackageList := deb.NewPackageList()
 	err = sourcePackageList.Append(packageList)
 	if err != nil {
-		fmt.Errorf("unable to merge sources: %s", err)
+		return fmt.Errorf("unable to merge sources: %s", err)
 	}
 
 	var pL *deb.PackageList
 	for i := 1; i < len(snapshots); i++ {
 		pL, err = deb.NewPackageListFromRefList(snapshots[i].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 		if err != nil {
-			fmt.Errorf("unable to load packages: %s", err)
+			return fmt.Errorf("unable to load packages: %s", err)
 		}
 
 		err = sourcePackageList.Append(pL)
 		if err != nil {
-			fmt.Errorf("unable to merge sources: %s", err)
+			return fmt.Errorf("unable to merge sources: %s", err)
 		}
 	}
 

cmd/task_run.go

@@ -20,7 +20,7 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 
 		var finfo os.FileInfo
 		if finfo, err = os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
-			return fmt.Errorf("no such file, %s\n", filename)
+			return fmt.Errorf("no such file, %s", filename)
 		}
 
 		fmt.Print("Reading file...\n\n")

cmd/version.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/commander"
 )

console/progress.go

@@ -2,10 +2,11 @@ package console
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/cheggaaa/pb"
 	"github.com/smira/aptly/aptly"
 	"github.com/wsxiaoys/terminal/color"
-	"strings"
 )
 
 const (

console/terminal.go

@@ -1,8 +1,9 @@
 package console
 
 import (
-	"golang.org/x/crypto/ssh/terminal"
 	"syscall"
+
+	"golang.org/x/crypto/ssh/terminal"
 )
 
 // RunningOnTerminal checks whether stdout is terminal

context/context.go

@@ -3,6 +3,14 @@ package context
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"strings"
+	"sync"
+	"time"
+
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/console"
 	"github.com/smira/aptly/database"
@@ -14,13 +22,6 @@ import (
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"path/filepath"
-	"runtime"
-	"runtime/pprof"
-	"strings"
-	"sync"
-	"time"
 )
 
 // AptlyContext is a common context shared by all commands
@@ -99,7 +100,7 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			}
 
 			if err != nil {
-				fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
+				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", configLocations[0])
 				utils.SaveConfig(configLocations[0], &utils.Config)
 			}
 		}
@@ -321,9 +322,11 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 			}
 
 			var err error
-			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
-				params.Region, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
-				params.EncryptionMethod, params.PlusWorkaround)
+			publishedStorage, err = s3.NewPublishedStorage(
+				params.AccessKeyID, params.SecretAccessKey, params.SessionToken,
+				params.Region, params.Endpoint, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
+				params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel,
+				params.ForceSigV2, params.Debug)
 			if err != nil {
 				Fatal(err)
 			}
@@ -335,7 +338,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 
 			var err error
 			publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password,
-				params.AuthURL, params.Tenant, params.TenantID, params.Container, params.Prefix)
+				params.AuthURL, params.Tenant, params.TenantID, params.Domain, params.DomainID, params.TenantDomain, params.TenantDomainID, params.Container, params.Prefix)
 			if err != nil {
 				Fatal(err)
 			}

database/leveldb.go

@@ -4,6 +4,9 @@ package database
 import (
 	"bytes"
 	"errors"
+	"io/ioutil"
+	"os"
+
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/filter"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -16,11 +19,17 @@ var (
 	ErrNotFound = errors.New("key not found")
 )
 
+// StorageProcessor is a function to process one single storage entry
+type StorageProcessor func(key []byte, value []byte) error
+
 // Storage is an interface to KV storage
 type Storage interface {
+	CreateTemporary() (Storage, error)
 	Get(key []byte) ([]byte, error)
 	Put(key []byte, value []byte) error
 	Delete(key []byte) error
+	HasPrefix(prefix []byte) bool
+	ProcessByPrefix(prefix []byte, proc StorageProcessor) error
 	KeysByPrefix(prefix []byte) [][]byte
 	FetchByPrefix(prefix []byte) [][]byte
 	Close() error
@@ -28,6 +37,7 @@ type Storage interface {
 	StartBatch()
 	FinishBatch() error
 	CompactDB() error
+	Drop() error
 }
 
 type levelDB struct {
@@ -41,9 +51,16 @@ var (
 	_ Storage = &levelDB{}
 )
 
-func internalOpen(path string) (*leveldb.DB, error) {
+func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) {
 	o := &opt.Options{
-		Filter: filter.NewBloomFilter(10),
+		Filter:                 filter.NewBloomFilter(10),
+		OpenFilesCacheCapacity: 256,
+	}
+
+	if throttleCompaction {
+		o.CompactionL0Trigger = 32
+		o.WriteL0PauseTrigger = 96
+		o.WriteL0SlowdownTrigger = 64
 	}
 
 	return leveldb.OpenFile(path, o)
@@ -51,7 +68,7 @@ func internalOpen(path string) (*leveldb.DB, error) {
 
 // OpenDB opens (creates) LevelDB database
 func OpenDB(path string) (Storage, error) {
-	db, err := internalOpen(path)
+	db, err := internalOpen(path, false)
 	if err != nil {
 		return nil, err
 	}
@@ -60,7 +77,7 @@ func OpenDB(path string) (Storage, error) {
 
 // RecoverDB recovers LevelDB database from corruption
 func RecoverDB(path string) error {
-	stor, err := storage.OpenFile(path)
+	stor, err := storage.OpenFile(path, false)
 	if err != nil {
 		return err
 	}
@@ -76,6 +93,20 @@ func RecoverDB(path string) error {
 	return nil
 }
 
+// CreateTemporary creates new DB of the same type in temp dir
+func (l *levelDB) CreateTemporary() (Storage, error) {
+	tempdir, err := ioutil.TempDir("", "aptly")
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := internalOpen(tempdir, true)
+	if err != nil {
+		return nil, err
+	}
+	return &levelDB{db: db, path: tempdir}, nil
+}
+
 // Get key value from database
 func (l *levelDB) Get(key []byte) ([]byte, error) {
 	value, err := l.db.Get(key, nil)
@@ -151,6 +182,29 @@ func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
 	return result
 }
 
+// HasPrefix checks whether it can find any key with given prefix and returns true if one exists
+func (l *levelDB) HasPrefix(prefix []byte) bool {
+	iterator := l.db.NewIterator(nil, nil)
+	defer iterator.Release()
+	return iterator.Seek(prefix) && bytes.HasPrefix(iterator.Key(), prefix)
+}
+
+// ProcessByPrefix iterates through all entries where key starts with prefix and calls
+// StorageProcessor on key value pair
+func (l *levelDB) ProcessByPrefix(prefix []byte, proc StorageProcessor) error {
+	iterator := l.db.NewIterator(nil, nil)
+	defer iterator.Release()
+
+	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
+		err := proc(iterator.Key(), iterator.Value())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // Close finishes DB work
 func (l *levelDB) Close() error {
 	if l.db == nil {
@@ -168,7 +222,7 @@ func (l *levelDB) ReOpen() error {
 	}
 
 	var err error
-	l.db, err = internalOpen(l.path)
+	l.db, err = internalOpen(l.path, false)
 	return err
 }
 
@@ -196,3 +250,12 @@ func (l *levelDB) FinishBatch() error {
 func (l *levelDB) CompactDB() error {
 	return l.db.CompactRange(util.Range{})
 }
+
+// Drop removes all the DB files (DANGEROUS!)
+func (l *levelDB) Drop() error {
+	if l.db != nil {
+		return errors.New("DB is still open")
+	}
+
+	return os.RemoveAll(l.path)
+}

database/leveldb_test.go

@@ -71,6 +71,29 @@ func (s *LevelDBSuite) TestGetPut(c *C) {
 	c.Assert(result, DeepEquals, value)
 }
 
+func (s *LevelDBSuite) TestTemporaryDelete(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	temp, err := s.db.CreateTemporary()
+	c.Assert(err, IsNil)
+
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
+	c.Check(temp.HasPrefix([]byte(nil)), Equals, false)
+
+	err = temp.Put(key, value)
+	c.Assert(err, IsNil)
+	c.Check(temp.HasPrefix([]byte(nil)), Equals, true)
+
+	c.Assert(temp.Close(), IsNil)
+	c.Assert(temp.Drop(), IsNil)
+}
+
 func (s *LevelDBSuite) TestDelete(c *C) {
 	var (
 		key   = []byte("key")
@@ -107,10 +130,41 @@ func (s *LevelDBSuite) TestByPrefix(c *C) {
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
+	keys := [][]byte{}
+	values := [][]byte{}
+
+	c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
+		keys = append(keys, append([]byte(nil), k...))
+		values = append(values, append([]byte(nil), v...))
+		return nil
+	}), IsNil)
+
+	c.Check(values, DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(keys, DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
+		return ErrNotFound
+	}), Equals, ErrNotFound)
+
+	c.Check(s.db.ProcessByPrefix([]byte{0xa0}, func(k, v []byte) error {
+		return ErrNotFound
+	}), IsNil)
+
 	c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
 	c.Check(s.db.KeysByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
 }
 
+func (s *LevelDBSuite) TestHasPrefix(c *C) {
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
+	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
+
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
+	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
+	c.Check(s.db.HasPrefix([]byte{0x79}), Equals, false)
+}
+
 func (s *LevelDBSuite) TestBatch(c *C) {
 	var (
 		key    = []byte("key")

deb/changes.go

@@ -0,0 +1,292 @@
+package deb
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+)
+
+// Changes is a result of .changes file parsing
+type Changes struct {
+	Changes               string
+	Distribution          string
+	Files                 PackageFiles
+	BasePath, ChangesName string
+	TempDir               string
+	Source                string
+	Binary                []string
+	Architectures         []string
+	Stanza                Stanza
+	SignatureKeys         []utils.GpgKey
+}
+
+// NewChanges moves .changes file into temporary directory and creates Changes structure
+func NewChanges(path string) (*Changes, error) {
+	var err error
+
+	c := &Changes{
+		BasePath:    filepath.Dir(path),
+		ChangesName: filepath.Base(path),
+	}
+
+	c.TempDir, err = ioutil.TempDir(os.TempDir(), "aptly")
+	if err != nil {
+		return nil, err
+	}
+
+	// copy .changes file into temporary directory
+	err = utils.CopyFile(filepath.Join(c.BasePath, c.ChangesName), filepath.Join(c.TempDir, c.ChangesName))
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// VerifyAndParse does optional signature verification and parses changes files
+func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier utils.Verifier) error {
+	input, err := os.Open(filepath.Join(c.TempDir, c.ChangesName))
+	if err != nil {
+		return err
+	}
+	defer input.Close()
+
+	isClearSigned, err := verifier.IsClearSigned(input)
+	if err != nil {
+		return err
+	}
+
+	input.Seek(0, 0)
+
+	if !isClearSigned && !acceptUnsigned {
+		return fmt.Errorf(".changes file is not signed and unsigned processing hasn't been enabled")
+	}
+
+	if isClearSigned && !ignoreSignature {
+		keyInfo, err := verifier.VerifyClearsigned(input, false)
+		if err != nil {
+			return err
+		}
+		input.Seek(0, 0)
+
+		c.SignatureKeys = keyInfo.GoodKeys
+	}
+
+	var text *os.File
+
+	if isClearSigned {
+		text, err = verifier.ExtractClearsigned(input)
+		if err != nil {
+			return err
+		}
+		defer text.Close()
+	} else {
+		text = input
+	}
+
+	reader := NewControlFileReader(text)
+	c.Stanza, err = reader.ReadStanza(false)
+	if err != nil {
+		return err
+	}
+
+	c.Distribution = c.Stanza["Distribution"]
+	c.Changes = c.Stanza["Changes"]
+	c.Source = c.Stanza["Source"]
+	c.Binary = strings.Fields(c.Stanza["Binary"])
+	c.Architectures = strings.Fields(c.Stanza["Architecture"])
+
+	c.Files, err = c.Files.ParseSumFields(c.Stanza)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Prepare creates temporary directory, copies file there and verifies checksums
+func (c *Changes) Prepare() error {
+	var err error
+
+	for _, file := range c.Files {
+		if filepath.Dir(file.Filename) != "." {
+			return fmt.Errorf("file is not in the same folder as .changes file: %s", file.Filename)
+		}
+
+		file.Filename = filepath.Base(file.Filename)
+
+		err = utils.CopyFile(filepath.Join(c.BasePath, file.Filename), filepath.Join(c.TempDir, file.Filename))
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, file := range c.Files {
+		var info utils.ChecksumInfo
+
+		info, err = utils.ChecksumsForFile(filepath.Join(c.TempDir, file.Filename))
+		if err != nil {
+			return err
+		}
+
+		if info.Size != file.Checksums.Size {
+			return fmt.Errorf("size mismatch: expected %v != obtained %v", file.Checksums.Size, info.Size)
+		}
+
+		if info.MD5 != file.Checksums.MD5 {
+			return fmt.Errorf("checksum mismatch MD5: expected %v != obtained %v", file.Checksums.MD5, info.MD5)
+		}
+
+		if info.SHA1 != file.Checksums.SHA1 {
+			return fmt.Errorf("checksum mismatch SHA1: expected %v != obtained %v", file.Checksums.SHA1, info.SHA1)
+		}
+
+		if info.SHA256 != file.Checksums.SHA256 {
+			return fmt.Errorf("checksum mismatch SHA256 expected %v != obtained %v", file.Checksums.SHA256, info.SHA256)
+		}
+	}
+
+	return nil
+}
+
+// Cleanup removes all temporary files
+func (c *Changes) Cleanup() error {
+	if c.TempDir == "" {
+		return nil
+	}
+
+	return os.RemoveAll(c.TempDir)
+}
+
+// PackageQuery returns query that every package should match to be included
+func (c *Changes) PackageQuery() (PackageQuery, error) {
+	var archQuery PackageQuery = &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: ""}
+	for _, arch := range c.Architectures {
+		archQuery = &OrQuery{L: &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: arch}, R: archQuery}
+	}
+
+	// if c.Source is empty, this would never match
+	sourceQuery := &AndQuery{
+		L: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"},
+		R: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Source},
+	}
+
+	var binaryQuery PackageQuery
+	if len(c.Binary) > 0 {
+		binaryQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Binary[0]}
+		// matching debug ddeb packages, they're not present in the Binary field
+		var ddebQuery PackageQuery
+		ddebQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: fmt.Sprintf("%s-dbgsym", c.Binary[0])}
+
+		for _, binary := range c.Binary[1:] {
+			binaryQuery = &OrQuery{
+				L: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: binary},
+				R: binaryQuery,
+			}
+			ddebQuery = &OrQuery{
+				L: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: fmt.Sprintf("%s-dbgsym", binary)},
+				R: ddebQuery,
+			}
+		}
+
+		ddebQuery = &AndQuery{
+			L: &FieldQuery{Field: "Source", Relation: VersionEqual, Value: c.Source},
+			R: ddebQuery,
+		}
+
+		binaryQuery = &OrQuery{
+			L: binaryQuery,
+			R: ddebQuery,
+		}
+
+		binaryQuery = &AndQuery{
+			L: &NotQuery{Q: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"}},
+			R: binaryQuery}
+	}
+
+	var nameQuery PackageQuery
+	if binaryQuery == nil {
+		nameQuery = sourceQuery
+	} else {
+		nameQuery = &OrQuery{L: sourceQuery, R: binaryQuery}
+	}
+
+	return &AndQuery{L: archQuery, R: nameQuery}, nil
+}
+
+// GetField implements PackageLike interface
+func (c *Changes) GetField(field string) string {
+	return c.Stanza[field]
+}
+
+// MatchesDependency implements PackageLike interface
+func (c *Changes) MatchesDependency(d Dependency) bool {
+	return false
+}
+
+// MatchesArchitecture implements PackageLike interface
+func (c *Changes) MatchesArchitecture(arch string) bool {
+	return false
+}
+
+// GetName implements PackageLike interface
+func (c *Changes) GetName() string {
+	return ""
+}
+
+// GetVersion implements PackageLike interface
+func (c *Changes) GetVersion() string {
+	return ""
+
+}
+
+// GetArchitecture implements PackageLike interface
+func (c *Changes) GetArchitecture() string {
+	return ""
+}
+
+// CollectChangesFiles walks filesystem collecting all .changes files
+func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (changesFiles, failedFiles []string) {
+	for _, location := range locations {
+		info, err2 := os.Stat(location)
+		if err2 != nil {
+			reporter.Warning("Unable to process %s: %s", location, err2)
+			failedFiles = append(failedFiles, location)
+			continue
+		}
+		if info.IsDir() {
+			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
+				if err3 != nil {
+					return err3
+				}
+				if info.IsDir() {
+					return nil
+				}
+
+				if strings.HasSuffix(info.Name(), ".changes") {
+					changesFiles = append(changesFiles, path)
+				}
+
+				return nil
+			})
+
+			if err2 != nil {
+				reporter.Warning("Unable to process %s: %s", location, err2)
+				failedFiles = append(failedFiles, location)
+				continue
+			}
+		} else if strings.HasSuffix(info.Name(), ".changes") {
+			changesFiles = append(changesFiles, location)
+		}
+	}
+
+	sort.Strings(changesFiles)
+
+	return
+}

deb/changes_test.go

@@ -0,0 +1,92 @@
+package deb
+
+import (
+	"os"
+	"path/filepath"
+
+	. "gopkg.in/check.v1"
+)
+
+type ChangesSuite struct {
+	Dir, Path string
+}
+
+var _ = Suite(&ChangesSuite{})
+
+func (s *ChangesSuite) SetUpTest(c *C) {
+	s.Dir = c.MkDir()
+	s.Path = filepath.Join(s.Dir, "calamares.changes")
+
+	f, err := os.Create(s.Path)
+	c.Assert(err, IsNil)
+
+	f.WriteString(changesFile)
+	f.Close()
+}
+
+func (s *ChangesSuite) TestParseAndVerify(c *C) {
+	changes, err := NewChanges(s.Path)
+	c.Assert(err, IsNil)
+
+	err = changes.VerifyAndParse(true, true, &NullVerifier{})
+	c.Check(err, IsNil)
+
+	c.Check(changes.Distribution, Equals, "sid")
+	c.Check(changes.Files, HasLen, 4)
+	c.Check(changes.Files[0].Filename, Equals, "calamares_0+git20141127.99.dsc")
+	c.Check(changes.Files[0].Checksums.Size, Equals, int64(1106))
+	c.Check(changes.Files[0].Checksums.MD5, Equals, "05fd8f3ffe8f362c5ef9bad2f936a56e")
+	c.Check(changes.Files[0].Checksums.SHA1, Equals, "79f10e955dab6eb25b7f7bae18213f367a3a0396")
+	c.Check(changes.Files[0].Checksums.SHA256, Equals, "35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc")
+	c.Check(changes.BasePath, Equals, s.Dir)
+	c.Check(changes.Architectures, DeepEquals, []string{"source", "amd64"})
+	c.Check(changes.Source, Equals, "calamares")
+	c.Check(changes.Binary, DeepEquals, []string{"calamares", "calamares-dbg"})
+}
+
+func (s *ChangesSuite) TestPackageQuery(c *C) {
+	changes, err := NewChanges(s.Path)
+	c.Assert(err, IsNil)
+
+	err = changes.VerifyAndParse(true, true, &NullVerifier{})
+	c.Check(err, IsNil)
+
+	q, err := changes.PackageQuery()
+	c.Check(err, IsNil)
+
+	c.Check(q.String(), Equals,
+		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), (((Name (= calamares-dbg)) | (Name (= calamares))) | ((Source (= calamares)), ((Name (= calamares-dbg-dbgsym)) | (Name (= calamares-dbgsym)))))))")
+}
+
+var changesFile = `Format: 1.8
+Date: Thu, 27 Nov 2014 13:24:53 +0000
+Source: calamares
+Binary: calamares calamares-dbg
+Architecture: source amd64
+Version: 0+git20141127.99
+Distribution: sid
+Urgency: medium
+Maintainer: Rohan Garg <rohan@kde.org>
+Changed-By: Rohan <rohan@kde.org>
+Description:
+ calamares  - distribution-independent installer framework
+ calamares-dbg - distribution-independent installer framework -- debug symbols
+Changes:
+ calamares (0+git20141127.99) sid; urgency=medium
+ .
+   * Update from git
+Checksums-Sha1:
+ 79f10e955dab6eb25b7f7bae18213f367a3a0396 1106 calamares_0+git20141127.99.dsc
+ 294c28e2c8e34e72ca9ee0d9da5c14f3bf4188db 2694800 calamares_0+git20141127.99.tar.xz
+ d6c26c04b5407c7511f61cb3e3de60c4a1d6c4ff 1698924 calamares_0+git20141127.99_amd64.deb
+ a3da632d193007b0d4a1aff73159fde1b532d7a8 12835902 calamares-dbg_0+git20141127.99_amd64.deb
+Checksums-Sha256:
+ 35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc 1106 calamares_0+git20141127.99.dsc
+ 5576b9caaf814564830f95561227e4f04ee87b31da22c1371aab155cbf7ce395 2694800 calamares_0+git20141127.99.tar.xz
+ 2e6e2f232ed7ffe52369928ebdf5436d90feb37840286ffba79e87d57a43a2e9 1698924 calamares_0+git20141127.99_amd64.deb
+ 8dd926080ed7bad2e2439e37e49ce12d5f1357c5041b7da4d860a1041f878a8a 12835902 calamares-dbg_0+git20141127.99_amd64.deb
+Files:
+ 05fd8f3ffe8f362c5ef9bad2f936a56e 1106 devel optional calamares_0+git20141127.99.dsc
+ 097e55c81abd8e5f30bb2eed90c2c1e9 2694800 devel optional calamares_0+git20141127.99.tar.xz
+ 827fb3b12534241e119815d331e8197b 1698924 devel optional calamares_0+git20141127.99_amd64.deb
+ e6f8ce70f564d1f68cb57758b15b13e3 12835902 debug optional calamares-dbg_0+git20141127.99_amd64.deb`

deb/collections.go

@@ -1,8 +1,9 @@
 package deb
 
 import (
-	"github.com/smira/aptly/database"
 	"sync"
+
+	"github.com/smira/aptly/database"
 )
 
 // CollectionFactory is a single place to generate all desired collections
@@ -21,6 +22,13 @@ func NewCollectionFactory(db database.Storage) *CollectionFactory {
 	return &CollectionFactory{Mutex: &sync.Mutex{}, db: db}
 }
 
+// TemporaryDB creates new temporary DB
+//
+// DB should be closed/droped after being used
+func (factory *CollectionFactory) TemporaryDB() (database.Storage, error) {
+	return factory.db.CreateTemporary()
+}
+
 // PackageCollection returns (or creates) new PackageCollection
 func (factory *CollectionFactory) PackageCollection() *PackageCollection {
 	factory.Lock()

deb/contents.go

@@ -0,0 +1,135 @@
+package deb
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/database"
+	"github.com/smira/go-uuid/uuid"
+)
+
+// ContentsIndex calculates mapping from files to packages, with sorting and aggregation
+type ContentsIndex struct {
+	db     database.Storage
+	prefix []byte
+}
+
+// NewContentsIndex creates empty ContentsIndex
+func NewContentsIndex(db database.Storage) *ContentsIndex {
+	return &ContentsIndex{
+		db:     db,
+		prefix: []byte(uuid.New()),
+	}
+}
+
+// Push adds package to contents index, calculating package contents as required
+func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool) error {
+	contents := p.Contents(packagePool)
+	qualifiedName := []byte(p.QualifiedName())
+
+	for _, path := range contents {
+		// for performance reasons we only write to leveldb during push.
+		// merging of qualified names per path will be done in WriteTo
+		err := index.db.Put(append(append(append(index.prefix, []byte(path)...), byte(0)), qualifiedName...), nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Empty checks whether index contains no packages
+func (index *ContentsIndex) Empty() bool {
+	return !index.db.HasPrefix(index.prefix)
+}
+
+// WriteTo dumps sorted mapping of files to qualified package names
+func (index *ContentsIndex) WriteTo(w io.Writer) (int64, error) {
+	// For performance reasons push method wrote on key per path and package
+	// in this method we now need to merge all packages which have the same path
+	// and write it to contents index file
+
+	var n int64
+
+	nn, err := fmt.Fprintf(w, "%s %s\n", "FILE", "LOCATION")
+	n += int64(nn)
+	if err != nil {
+		return n, err
+	}
+
+	prefixLen := len(index.prefix)
+
+	var (
+		currentPath []byte
+		currentPkgs [][]byte
+	)
+
+	err = index.db.ProcessByPrefix(index.prefix, func(key []byte, value []byte) error {
+		// cut prefix
+		key = key[prefixLen:]
+
+		i := bytes.Index(key, []byte{0})
+		if i == -1 {
+			return errors.New("corrupted index entry")
+		}
+
+		path := key[:i]
+		pkg := key[i+1:]
+
+		if !bytes.Equal(path, currentPath) {
+			if currentPath != nil {
+				nn, err = w.Write(append(currentPath, ' '))
+				n += int64(nn)
+				if err != nil {
+					return err
+				}
+
+				nn, err = w.Write(bytes.Join(currentPkgs, []byte{','}))
+				n += int64(nn)
+				if err != nil {
+					return err
+				}
+
+				nn, err = w.Write([]byte{'\n'})
+				n += int64(nn)
+				if err != nil {
+					return err
+				}
+			}
+
+			currentPath = append([]byte(nil), path...)
+			currentPkgs = nil
+		}
+
+		currentPkgs = append(currentPkgs, append([]byte(nil), pkg...))
+
+		return nil
+	})
+
+	if err != nil {
+		return n, err
+	}
+
+	if currentPath != nil {
+		nn, err = w.Write(append(currentPath, ' '))
+		n += int64(nn)
+		if err != nil {
+			return n, err
+		}
+
+		nn, err = w.Write(bytes.Join(currentPkgs, []byte{','}))
+		n += int64(nn)
+		if err != nil {
+			return n, err
+		}
+
+		nn, err = w.Write([]byte{'\n'})
+		n += int64(nn)
+	}
+
+	return n, err
+}

deb/deb.go

@@ -3,13 +3,19 @@ package deb
 import (
 	"archive/tar"
 	"bufio"
+	"compress/bzip2"
 	"compress/gzip"
 	"fmt"
-	"github.com/mkrautz/goar"
-	"github.com/smira/aptly/utils"
 	"io"
 	"os"
 	"strings"
+
+	"github.com/h2non/filetype/matchers"
+	"github.com/mkrautz/goar"
+
+	"github.com/smira/aptly/utils"
+	"github.com/smira/go-xz"
+	"github.com/smira/lzma"
 )
 
 // GetControlFileFromDeb reads control file from deb package
@@ -24,16 +30,16 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	for {
 		header, err := library.Next()
 		if err == io.EOF {
-			return nil, fmt.Errorf("unable to find control.tar.gz part")
+			return nil, fmt.Errorf("unable to find control.tar.gz part in package %s", packageFile)
 		}
 		if err != nil {
-			return nil, fmt.Errorf("unable to read .deb archive: %s", err)
+			return nil, fmt.Errorf("unable to read .deb archive %s: %s", packageFile, err)
 		}
 
 		if header.Name == "control.tar.gz" {
 			ungzip, err := gzip.NewReader(library)
 			if err != nil {
-				return nil, fmt.Errorf("unable to ungzip: %s", err)
+				return nil, fmt.Errorf("unable to ungzip control file from %s. Error: %s", packageFile, err)
 			}
 			defer ungzip.Close()
 
@@ -41,15 +47,15 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 			for {
 				tarHeader, err := untar.Next()
 				if err == io.EOF {
-					return nil, fmt.Errorf("unable to find control file")
+					return nil, fmt.Errorf("unable to find control file in %s", packageFile)
 				}
 				if err != nil {
-					return nil, fmt.Errorf("unable to read .tar archive: %s", err)
+					return nil, fmt.Errorf("unable to read .tar archive from %s. Error: %s", packageFile, err)
 				}
 
 				if tarHeader.Name == "./control" || tarHeader.Name == "control" {
 					reader := NewControlFileReader(untar)
-					stanza, err := reader.ReadStanza()
+					stanza, err := reader.ReadStanza(false)
 					if err != nil {
 						return nil, err
 					}
@@ -69,16 +75,16 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 	}
 	defer file.Close()
 
-	line, err := bufio.NewReader(file).ReadString('\n')
+	isClearSigned, err := verifier.IsClearSigned(file)
+	file.Seek(0, 0)
+
 	if err != nil {
 		return nil, err
 	}
 
-	file.Seek(0, 0)
-
 	var text *os.File
 
-	if strings.Index(line, "BEGIN PGP SIGN") != -1 {
+	if isClearSigned {
 		text, err = verifier.ExtractClearsigned(file)
 		if err != nil {
 			return nil, err
@@ -89,7 +95,7 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 	}
 
 	reader := NewControlFileReader(text)
-	stanza, err := reader.ReadStanza()
+	stanza, err := reader.ReadStanza(false)
 	if err != nil {
 		return nil, err
 	}
@@ -97,3 +103,87 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 	return stanza, nil
 
 }
+
+// GetContentsFromDeb returns list of files installed by .deb package
+func GetContentsFromDeb(packageFile string) ([]string, error) {
+	file, err := os.Open(packageFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	library := ar.NewReader(file)
+	for {
+		header, err := library.Next()
+		if err == io.EOF {
+			return nil, fmt.Errorf("unable to find data.tar.* part in %s", packageFile)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("unable to read .deb archive from %s: %s", packageFile, err)
+		}
+
+		if strings.HasPrefix(header.Name, "data.tar") {
+			bufReader := bufio.NewReader(library)
+			signature, err := bufReader.Peek(270)
+
+			var isTar bool
+			if err == nil {
+				isTar = matchers.Tar(signature)
+			}
+
+			var tarInput io.Reader
+
+			switch header.Name {
+			case "data.tar":
+				tarInput = bufReader
+			case "data.tar.gz":
+				if isTar {
+					tarInput = bufReader
+				} else {
+					ungzip, err := gzip.NewReader(bufReader)
+					if err != nil {
+						return nil, fmt.Errorf("unable to ungzip data.tar.gz from %s: %s", packageFile, err)
+					}
+					defer ungzip.Close()
+					tarInput = ungzip
+				}
+			case "data.tar.bz2":
+				tarInput = bzip2.NewReader(bufReader)
+			case "data.tar.xz":
+				unxz, err := xz.NewReader(bufReader)
+				if err != nil {
+					return nil, fmt.Errorf("unable to unxz data.tar.xz from %s: %s", packageFile, err)
+				}
+				defer unxz.Close()
+				tarInput = unxz
+			case "data.tar.lzma":
+				unlzma := lzma.NewReader(bufReader)
+				defer unlzma.Close()
+				tarInput = unlzma
+			default:
+				return nil, fmt.Errorf("unsupported tar compression in %s: %s", packageFile, header.Name)
+			}
+
+			untar := tar.NewReader(tarInput)
+			var results []string
+			for {
+				tarHeader, err := untar.Next()
+				if err == io.EOF {
+					return results, nil
+				}
+				if err != nil {
+					return nil, fmt.Errorf("unable to read .tar archive from %s: %s", packageFile, err)
+				}
+
+				if tarHeader.Typeflag == tar.TypeDir {
+					continue
+				}
+
+				if strings.HasPrefix(tarHeader.Name, "./") {
+					tarHeader.Name = tarHeader.Name[2:]
+				}
+				results = append(results, tarHeader.Name)
+			}
+		}
+	}
+}

deb/deb_test.go

@@ -1,15 +1,16 @@
 package deb
 
 import (
-	"github.com/smira/aptly/utils"
 	"path/filepath"
 	"runtime"
 
+	"github.com/smira/aptly/utils"
+
 	. "gopkg.in/check.v1"
 )
 
 type DebSuite struct {
-	debFile, dscFile, dscFileNoSign string
+	debFile, debFile2, dscFile, dscFileNoSign string
 }
 
 var _ = Suite(&DebSuite{})
@@ -17,6 +18,7 @@ var _ = Suite(&DebSuite{})
 func (s *DebSuite) SetUpSuite(c *C) {
 	_, _File, _, _ := runtime.Caller(0)
 	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	s.debFile2 = filepath.Join(filepath.Dir(_File), "../system/changes/hardlink_0.2.1_amd64.deb")
 	s.dscFile = filepath.Join(filepath.Dir(_File), "../system/files/pyspi_0.6.1-1.3.dsc")
 	s.dscFileNoSign = filepath.Join(filepath.Dir(_File), "../system/files/pyspi-0.6.1-1.3.stripped.dsc")
 }
@@ -27,7 +29,7 @@ func (s *DebSuite) TestGetControlFileFromDeb(c *C) {
 
 	_, _File, _, _ := runtime.Caller(0)
 	_, err = GetControlFileFromDeb(_File)
-	c.Check(err, ErrorMatches, "unable to read .deb archive: ar: missing global header")
+	c.Check(err, ErrorMatches, "^.+ar: missing global header")
 
 	st, err := GetControlFileFromDeb(s.debFile)
 	c.Check(err, IsNil)
@@ -55,3 +57,15 @@ func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
 	c.Check(st["Version"], Equals, "0.6.1-1.4")
 	c.Check(st["Source"], Equals, "pyspi")
 }
+
+func (s *DebSuite) TestGetContentsFromDeb(c *C) {
+	contents, err := GetContentsFromDeb(s.debFile)
+	c.Check(err, IsNil)
+	c.Check(contents, DeepEquals, []string{"usr/share/doc/libboost-program-options-dev/changelog.gz",
+		"usr/share/doc/libboost-program-options-dev/copyright"})
+
+	contents, err = GetContentsFromDeb(s.debFile2)
+	c.Check(err, IsNil)
+	c.Check(contents, DeepEquals, []string{"usr/bin/hardlink", "usr/share/man/man1/hardlink.1.gz",
+		"usr/share/doc/hardlink/changelog.gz", "usr/share/doc/hardlink/copyright", "usr/share/doc/hardlink/NEWS.Debian.gz"})
+}

deb/format.go

@@ -30,6 +30,7 @@ var (
 		"MD5Sum",
 		"SHA1",
 		"SHA256",
+		"SHA512",
 	}
 
 	canonicalOrderBinary = []string{
@@ -59,6 +60,7 @@ var (
 		"MD5sum",
 		"SHA1",
 		"SHA256",
+		"SHA512",
 		"Description",
 	}
 
@@ -92,16 +94,46 @@ func (s Stanza) Copy() (result Stanza) {
 	return
 }
 
-// Write single field from Stanza to writer
-func writeField(w *bufio.Writer, field, value string) (err error) {
-	_, multiline := multilineFields[field]
+func isMultilineField(field string, isRelease bool) bool {
+	switch field {
+	case "Description":
+		return true
+	case "Files":
+		return true
+	case "Changes":
+		return true
+	case "Checksums-Sha1":
+		return true
+	case "Checksums-Sha256":
+		return true
+	case "Checksums-Sha512":
+		return true
+	case "Package-List":
+		return true
+	case "MD5Sum":
+		return isRelease
+	case "SHA1":
+		return isRelease
+	case "SHA256":
+		return isRelease
+	case "SHA512":
+		return isRelease
+	}
+	return false
+}
 
-	if !multiline {
+// Write single field from Stanza to writer
+func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) {
+	if !isMultilineField(field, isRelease) {
 		_, err = w.WriteString(field + ": " + value + "\n")
 	} else {
 		if !strings.HasSuffix(value, "\n") {
 			value = value + "\n"
 		}
+
+		if field != "Description" {
+			value = "\n" + value
+		}
 		_, err = w.WriteString(field + ":" + value)
 	}
 
@@ -122,7 +154,7 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 		value, ok := s[field]
 		if ok {
 			delete(s, field)
-			err := writeField(w, field, value)
+			err := writeField(w, field, value, isRelease)
 			if err != nil {
 				return err
 			}
@@ -130,7 +162,7 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 	}
 
 	for field, value := range s {
-		err := writeField(w, field, value)
+		err := writeField(w, field, value, isRelease)
 		if err != nil {
 			return err
 		}
@@ -144,20 +176,6 @@ var (
 	ErrMalformedStanza = errors.New("malformed stanza syntax")
 )
 
-var multilineFields = make(map[string]bool)
-
-func init() {
-	multilineFields["Description"] = true
-	multilineFields["Files"] = true
-	multilineFields["Changes"] = true
-	multilineFields["Checksums-Sha1"] = true
-	multilineFields["Checksums-Sha256"] = true
-	multilineFields["Package-List"] = true
-	multilineFields["SHA256"] = true
-	multilineFields["SHA1"] = true
-	multilineFields["MD5Sum"] = true
-}
-
 func canonicalCase(field string) string {
 	upper := strings.ToUpper(field)
 	switch upper {
@@ -198,7 +216,7 @@ func NewControlFileReader(r io.Reader) *ControlFileReader {
 }
 
 // ReadStanza reeads one stanza from control file
-func (c *ControlFileReader) ReadStanza() (Stanza, error) {
+func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
 	stanza := make(Stanza, 32)
 	lastField := ""
 	lastFieldMultiline := false
@@ -218,7 +236,7 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 			if lastFieldMultiline {
 				stanza[lastField] += line + "\n"
 			} else {
-				stanza[lastField] += strings.TrimSpace(line)
+				stanza[lastField] += " " + strings.TrimSpace(line)
 			}
 		} else {
 			parts := strings.SplitN(line, ":", 2)
@@ -226,7 +244,7 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 				return nil, ErrMalformedStanza
 			}
 			lastField = canonicalCase(parts[0])
-			_, lastFieldMultiline = multilineFields[lastField]
+			lastFieldMultiline = isMultilineField(lastField, isRelease)
 			if lastFieldMultiline {
 				stanza[lastField] = parts[1]
 				if parts[1] != "" {

deb/format_test.go

@@ -84,18 +84,18 @@ func (s *ControlFileSuite) SetUpTest(c *C) {
 func (s *ControlFileSuite) TestReadStanza(c *C) {
 	r := NewControlFileReader(s.reader)
 
-	stanza1, err := r.ReadStanza()
+	stanza1, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
-	stanza2, err := r.ReadStanza()
+	stanza2, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
-	stanza3, err := r.ReadStanza()
+	stanza3, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 	c.Assert(stanza3, IsNil)
 
 	c.Check(stanza1["Format"], Equals, "3.0 (quilt)")
-	c.Check(stanza1["Build-Depends"], Equals, "debhelper (>= 8),bash-completion (>= 1:1.1-3),libcurl4-nss-dev, libreadline-dev, libxml2-dev, libpcre3-dev, liboauth-dev, xsltproc, docbook-xsl, docbook-xml, dh-autoreconf")
+	c.Check(stanza1["Build-Depends"], Equals, "debhelper (>= 8), bash-completion (>= 1:1.1-3), libcurl4-nss-dev, libreadline-dev, libxml2-dev, libpcre3-dev, liboauth-dev, xsltproc, docbook-xsl, docbook-xml, dh-autoreconf")
 	c.Check(stanza1["Files"], Equals, " 3d5f65778bf3f89be03c313b0024b62c 1980 bti_032-1.dsc\n"+
 		" 1e0d0b693fdeebec268004ba41701baf 59773 bti_032.orig.tar.gz\n"+" ac1229a6d685023aeb8fcb0806324aa8 5065 bti_032-1.debian.tar.gz\n")
 	c.Check(len(stanza2), Equals, 20)
@@ -103,12 +103,12 @@ func (s *ControlFileSuite) TestReadStanza(c *C) {
 
 func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 	r := NewControlFileReader(s.reader)
-	stanza, err := r.ReadStanza()
+	stanza, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
 	buf := &bytes.Buffer{}
 	w := bufio.NewWriter(buf)
-	err = stanza.Copy().WriteTo(w, false, false)
+	err = stanza.Copy().WriteTo(w, true, false)
 	c.Assert(err, IsNil)
 	err = w.Flush()
 	c.Assert(err, IsNil)
@@ -116,7 +116,7 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 	str := buf.String()
 
 	r = NewControlFileReader(buf)
-	stanza2, err := r.ReadStanza()
+	stanza2, err := r.ReadStanza(false)
 	c.Assert(err, IsNil)
 
 	c.Assert(stanza2, DeepEquals, stanza)
@@ -140,7 +140,7 @@ func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
 		reader := bytes.NewBufferString(controlFile)
 		r := NewControlFileReader(reader)
 		for {
-			s, e := r.ReadStanza()
+			s, e := r.ReadStanza(false)
 			if s == nil && e == nil {
 				break
 			}

deb/graph.go

@@ -1,19 +1,35 @@
 package deb
 
 import (
-	"code.google.com/p/gographviz"
 	"fmt"
 	"strings"
+
+	"github.com/awalterschulze/gographviz"
 )
 
 // BuildGraph generates graph contents from aptly object database
-func BuildGraph(collectionFactory *CollectionFactory) (gographviz.Interface, error) {
+func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz.Interface, error) {
 	var err error
 
 	graph := gographviz.NewEscape()
 	graph.SetDir(true)
 	graph.SetName("aptly")
 
+	var labelStart string
+	var labelEnd string
+
+	switch layout {
+	case "vertical":
+		graph.AddAttr("aptly", "rankdir", "LR")
+		labelStart = ""
+		labelEnd = ""
+	case "horizontal":
+		fallthrough
+	default:
+		labelStart = "{"
+		labelEnd = "}"
+	}
+
 	existingNodes := map[string]bool{}
 
 	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *RemoteRepo) error {
@@ -26,9 +42,9 @@ func BuildGraph(collectionFactory *CollectionFactory) (gographviz.Interface, err
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "darkgoldenrod1",
-			"label": fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
-				repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "),
-				strings.Join(repo.Architectures, ", "), repo.NumPackages()),
+			"label": fmt.Sprintf("%sMirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d%s", labelStart, repo.Name, repo.ArchiveRoot,
+				repo.Distribution, strings.Join(repo.Components, ", "),
+				strings.Join(repo.Architectures, ", "), repo.NumPackages(), labelEnd),
 		})
 		existingNodes[repo.UUID] = true
 		return nil
@@ -48,8 +64,8 @@ func BuildGraph(collectionFactory *CollectionFactory) (gographviz.Interface, err
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "mediumseagreen",
-			"label": fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
-				repo.Name, repo.Comment, repo.NumPackages()),
+			"label": fmt.Sprintf("%sRepo %s|comment: %s|pkgs: %d%s", labelStart,
+				repo.Name, repo.Comment, repo.NumPackages(), labelEnd),
 		})
 		existingNodes[repo.UUID] = true
 		return nil
@@ -79,7 +95,8 @@ func BuildGraph(collectionFactory *CollectionFactory) (gographviz.Interface, err
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "cadetblue1",
-			"label":     fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages()),
+			"label": fmt.Sprintf("%sSnapshot %s|%s|pkgs: %d%s", labelStart,
+				snapshot.Name, description, snapshot.NumPackages(), labelEnd),
 		})
 
 		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
@@ -102,8 +119,9 @@ func BuildGraph(collectionFactory *CollectionFactory) (gographviz.Interface, err
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "darkolivegreen1",
-			"label": fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution,
-				strings.Join(repo.Components(), " "), strings.Join(repo.Architectures, ", ")),
+			"label": fmt.Sprintf("%sPublished %s/%s|comp: %s|arch: %s%s", labelStart,
+				repo.Prefix, repo.Distribution, strings.Join(repo.Components(), " "),
+				strings.Join(repo.Architectures, ", "), labelEnd),
 		})
 
 		for _, uuid := range repo.Sources {

deb/import.go

@@ -1,16 +1,17 @@
 package deb
 
 import (
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/utils"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
 )
 
 // CollectPackageFiles walks filesystem collecting all candidates for package files
-func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, failedFiles []string, err error) {
+func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, failedFiles []string) {
 	for _, location := range locations {
 		info, err2 := os.Stat(location)
 		if err2 != nil {
@@ -28,15 +29,21 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 				}
 
 				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
-					strings.HasSuffix(info.Name(), ".dsc") {
+					strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
 					packageFiles = append(packageFiles, path)
 				}
 
 				return nil
 			})
+
+			if err2 != nil {
+				reporter.Warning("Unable to process %s: %s", location, err2)
+				failedFiles = append(failedFiles, location)
+				continue
+			}
 		} else {
 			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
-				strings.HasSuffix(info.Name(), ".dsc") {
+				strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
 				packageFiles = append(packageFiles, location)
 			} else {
 				reporter.Warning("Unknown file extension: %s", location)
@@ -53,7 +60,7 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 
 // ImportPackageFiles imports files into local repository
 func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier utils.Verifier,
-	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter) (processedFiles []string, failedFiles []string, err error) {
+	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter, restriction PackageQuery) (processedFiles []string, failedFiles []string, err error) {
 	if forceReplace {
 		list.PrepareIndex()
 	}
@@ -150,6 +157,12 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
+		if restriction != nil && !restriction.Matches(p) {
+			reporter.Warning("%s has been ignored as it doesn't match restriction", p)
+			failedFiles = append(failedFiles, file)
+			continue
+		}
+
 		err = collection.Update(p)
 		if err != nil {
 			reporter.Warning("Unable to save package %s: %s", p, err)

deb/index_files.go

@@ -3,11 +3,12 @@ package deb
 import (
 	"bufio"
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/utils"
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
 )
 
 type indexFiles struct {
@@ -24,6 +25,7 @@ type indexFile struct {
 	parent       *indexFiles
 	discardable  bool
 	compressable bool
+	onlyGzip     bool
 	signable     bool
 	relativePath string
 	tempFilename string
@@ -61,7 +63,7 @@ func (file *indexFile) Finalize(signer utils.Signer) error {
 	}
 
 	if file.compressable {
-		err = utils.CompressFile(file.tempFile)
+		err = utils.CompressFile(file.tempFile, file.onlyGzip)
 		if err != nil {
 			file.tempFile.Close()
 			return fmt.Errorf("unable to compress index file: %s", err)
@@ -73,6 +75,9 @@ func (file *indexFile) Finalize(signer utils.Signer) error {
 	exts := []string{""}
 	if file.compressable {
 		exts = append(exts, ".gz", ".bz2")
+		if file.onlyGzip {
+			exts = []string{".gz"}
+		}
 	}
 
 	for _, ext := range exts {
@@ -215,6 +220,36 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 	return file
 }
 
+func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *indexFile {
+	if arch == "source" {
+		udeb = false
+	}
+	key := fmt.Sprintf("ci-%s-%s-%v", component, arch, udeb)
+	file, ok := files.indexes[key]
+	if !ok {
+		var relativePath string
+
+		if udeb {
+			relativePath = filepath.Join(component, fmt.Sprintf("Contents-udeb-%s", arch))
+		} else {
+			relativePath = filepath.Join(component, fmt.Sprintf("Contents-%s", arch))
+		}
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  true,
+			compressable: true,
+			onlyGzip:     true,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
 func (files *indexFiles) ReleaseFile() *indexFile {
 	return &indexFile{
 		parent:       files,

deb/list.go

@@ -2,9 +2,10 @@ package deb
 
 import (
 	"fmt"
+	"sort"
+
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
-	"sort"
 )
 
 // Dependency options
@@ -36,6 +37,10 @@ type PackageList struct {
 	packagesIndex []*Package
 	// Map of packages for each virtual package (provides)
 	providesIndex map[string][]*Package
+	// Package key generation function
+	keyFunc func(p *Package) string
+	// Allow duplicates?
+	duplicatesAllowed bool
 }
 
 // PackageConflictError means that package can't be added to the list due to error
@@ -49,9 +54,36 @@ var (
 	_ PackageCatalog = &PackageList{}
 )
 
-// NewPackageList creates empty package list
+func packageShortKey(p *Package) string {
+	return string(p.ShortKey(""))
+}
+
+func packageFullKey(p *Package) string {
+	return string(p.Key(""))
+}
+
+// NewPackageList creates empty package list without duplicate package
 func NewPackageList() *PackageList {
-	return &PackageList{packages: make(map[string]*Package, 1000)}
+	return NewPackageListWithDuplicates(false, 1000)
+}
+
+// NewPackageListWithDuplicates creates empty package list which might allow or block duplicate packages
+func NewPackageListWithDuplicates(duplicates bool, capacity int) *PackageList {
+	if capacity == 0 {
+		capacity = 1000
+	}
+
+	result := &PackageList{
+		packages:          make(map[string]*Package, capacity),
+		duplicatesAllowed: duplicates,
+		keyFunc:           packageShortKey,
+	}
+
+	if duplicates {
+		result.keyFunc = packageFullKey
+	}
+
+	return result
 }
 
 // NewPackageListFromRefList loads packages list from PackageRefList
@@ -61,7 +93,7 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 		return NewPackageList(), nil
 	}
 
-	result := &PackageList{packages: make(map[string]*Package, reflist.Len())}
+	result := NewPackageListWithDuplicates(false, reflist.Len())
 
 	if progress != nil {
 		progress.InitBar(int64(reflist.Len()), false)
@@ -91,7 +123,7 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 
 // Add appends package to package list, additionally checking for uniqueness
 func (l *PackageList) Add(p *Package) error {
-	key := string(p.ShortKey(""))
+	key := l.keyFunc(p)
 	existing, ok := l.packages[key]
 	if ok {
 		if !existing.Equals(p) {
@@ -170,7 +202,7 @@ func (l *PackageList) Append(pl *PackageList) error {
 
 // Remove removes package from the list, and updates index when required
 func (l *PackageList) Remove(p *Package) {
-	delete(l.packages, string(p.ShortKey("")))
+	delete(l.packages, l.keyFunc(p))
 	if l.indexed {
 		for _, provides := range p.Provides {
 			for i, pkg := range l.providesIndex[provides] {
@@ -217,7 +249,7 @@ func (l *PackageList) Strings() []string {
 
 	for _, p := range l.packages {
 		result[i] = string(p.Key(""))
-		i += 1
+		i++
 	}
 
 	return result
@@ -252,7 +284,7 @@ func depSliceDeduplicate(s []Dependency) []Dependency {
 
 // VerifyDependencies looks for missing dependencies in package list.
 //
-// Analysis would be peformed for each architecture, in specified sources
+// Analysis would be performed for each architecture, in specified sources
 func (l *PackageList) VerifyDependencies(options int, architectures []string, sources *PackageList, progress aptly.Progress) ([]Dependency, error) {
 	l.PrepareIndex()
 	missing := make([]Dependency, 0, 128)
@@ -365,7 +397,7 @@ func (l *PackageList) PrepareIndex() {
 
 // Scan searches package index using full scan
 func (l *PackageList) Scan(q PackageQuery) (result *PackageList) {
-	result = NewPackageList()
+	result = NewPackageListWithDuplicates(l.duplicatesAllowed, 0)
 	for _, pkg := range l.packages {
 		if q.Matches(pkg) {
 			result.Add(pkg)
@@ -382,7 +414,7 @@ func (l *PackageList) SearchSupported() bool {
 
 // SearchByKey looks up package by exact key reference
 func (l *PackageList) SearchByKey(arch, name, version string) (result *PackageList) {
-	result = NewPackageList()
+	result = NewPackageListWithDuplicates(l.duplicatesAllowed, 0)
 
 	pkg := l.packages["P"+arch+" "+name+" "+version]
 	if pkg != nil {

deb/list_test.go

@@ -379,6 +379,20 @@ func (s *PackageListSuite) TestFilter(c *C) {
 		&FieldQuery{Field: "$Architecture", Relation: VersionRegexp, Value: "i.*6", Regexp: regexp.MustCompile("i.*6")}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil)
 	c.Check(err, IsNil)
 	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")},
+		&NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}},
+	}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386")
+
+	result, err = s.il.Filter([]PackageQuery{&AndQuery{
+		&NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}},
+		&FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")},
+	}}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386")
 }
 
 func (s *PackageListSuite) TestVerifyDependencies(c *C) {

deb/local.go

@@ -2,12 +2,13 @@ package deb
 
 import (
 	"bytes"
-	"code.google.com/p/go-uuid/uuid"
 	"fmt"
-	"github.com/smira/aptly/database"
-	"github.com/ugorji/go/codec"
 	"log"
 	"sync"
+
+	"github.com/smira/aptly/database"
+	"github.com/smira/go-uuid/uuid"
+	"github.com/ugorji/go/codec"
 )
 
 // LocalRepo is a collection of packages created locally
@@ -22,6 +23,8 @@ type LocalRepo struct {
 	DefaultDistribution string `codec:",omitempty"`
 	// DefaultComponent
 	DefaultComponent string `codec:",omitempty"`
+	// Uploaders configuration
+	Uploaders *Uploaders `code:",omitempty" json:"-"`
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 }

deb/local_test.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"errors"
+
 	"github.com/smira/aptly/database"
 
 	. "gopkg.in/check.v1"

deb/package.go

@@ -3,11 +3,12 @@ package deb
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/utils"
 	"path/filepath"
 	"strconv"
 	"strings"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
 )
 
 // Package is single instance of Debian package
@@ -32,9 +33,10 @@ type Package struct {
 	// Is this >= 0.6 package?
 	V06Plus bool
 	// Offload fields
-	deps  *PackageDependencies
-	extra *Stanza
-	files *PackageFiles
+	deps     *PackageDependencies
+	extra    *Stanza
+	files    *PackageFiles
+	contents []string
 	// Mother collection
 	collection *PackageCollection
 }
@@ -75,6 +77,7 @@ func NewPackageFromControlFile(input Stanza) *Package {
 			MD5:    strings.TrimSpace(md5),
 			SHA1:   strings.TrimSpace(input["SHA1"]),
 			SHA256: strings.TrimSpace(input["SHA256"]),
+			SHA512: strings.TrimSpace(input["SHA512"]),
 		},
 	}})
 
@@ -83,6 +86,7 @@ func NewPackageFromControlFile(input Stanza) *Package {
 	delete(input, "MD5Sum")
 	delete(input, "SHA1")
 	delete(input, "SHA256")
+	delete(input, "SHA512")
 	delete(input, "Size")
 
 	depends := &PackageDependencies{}
@@ -114,62 +118,20 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 	delete(input, "Version")
 	delete(input, "Architecture")
 
+	var err error
+
 	files := make(PackageFiles, 0, 3)
-
-	parseSums := func(field string, setter func(sum *utils.ChecksumInfo, data string)) error {
-		for _, line := range strings.Split(input[field], "\n") {
-			line = strings.TrimSpace(line)
-			if line == "" {
-				continue
-			}
-			parts := strings.Fields(line)
-
-			if len(parts) != 3 {
-				return fmt.Errorf("unparseable hash sum line: %#v", line)
-			}
-
-			size, err := strconv.ParseInt(parts[1], 10, 64)
-			if err != nil {
-				return fmt.Errorf("unable to parse size: %s", err)
-			}
-
-			filename := filepath.Base(parts[2])
-
-			found := false
-			pos := 0
-			for i, file := range files {
-				if file.Filename == filename {
-					found = true
-					pos = i
-					break
-				}
-			}
-
-			if !found {
-				files = append(files, PackageFile{Filename: filename, downloadPath: input["Directory"]})
-				pos = len(files) - 1
-			}
-
-			files[pos].Checksums.Size = size
-			setter(&files[pos].Checksums, parts[0])
-		}
-
-		delete(input, field)
-
-		return nil
-	}
-
-	err := parseSums("Files", func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
+	files, err = files.ParseSumFields(input)
 	if err != nil {
 		return nil, err
 	}
-	err = parseSums("Checksums-Sha1", func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
-	if err != nil {
-		return nil, err
-	}
-	err = parseSums("Checksums-Sha256", func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
-	if err != nil {
-		return nil, err
+
+	delete(input, "Files")
+	delete(input, "Checksums-Sha1")
+	delete(input, "Checksums-Sha256")
+
+	for i := range files {
+		files[i].downloadPath = input["Directory"]
 	}
 
 	result.UpdateFiles(files)
@@ -211,14 +173,19 @@ func (p *Package) String() string {
 	return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.Architecture)
 }
 
-// MarshalJSON implements json.Marshaller interface
-func (p *Package) MarshalJSON() ([]byte, error) {
+// ExtendedStanza returns package stanza enhanced with aptly-specific fields
+func (p *Package) ExtendedStanza() Stanza {
 	stanza := p.Stanza()
 	stanza["FilesHash"] = fmt.Sprintf("%08x", p.FilesHash)
 	stanza["Key"] = string(p.Key(""))
 	stanza["ShortKey"] = string(p.ShortKey(""))
 
-	return json.Marshal(stanza)
+	return stanza
+}
+
+// MarshalJSON implements json.Marshaller interface
+func (p *Package) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.ExtendedStanza())
 }
 
 // GetField returns fields from package
@@ -336,7 +303,22 @@ func (p *Package) MatchesDependency(dep Dependency) bool {
 	panic("unknown relation")
 }
 
-// GetDependencies compiles list of dependenices by flags from options
+// GetName returns package name
+func (p *Package) GetName() string {
+	return p.Name
+}
+
+// GetVersion returns package version
+func (p *Package) GetVersion() string {
+	return p.Version
+}
+
+// GetArchitecture returns package arch
+func (p *Package) GetArchitecture() string {
+	return p.Architecture
+}
+
+// GetDependencies compiles list of dependncies by flags from options
 func (p *Package) GetDependencies(options int) (dependencies []string) {
 	deps := p.Deps()
 
@@ -372,6 +354,16 @@ func (p *Package) GetDependencies(options int) (dependencies []string) {
 	return
 }
 
+// QualifiedName returns [$SECTION/]$NAME
+func (p *Package) QualifiedName() string {
+	section := p.Extra()["Section"]
+	if section != "" {
+		return section + "/" + p.Name
+	}
+
+	return p.Name
+}
+
 // Extra returns Stanza of extra fields (it may load it from collection)
 func (p *Package) Extra() Stanza {
 	if p.extra == nil {
@@ -410,6 +402,35 @@ func (p *Package) Files() PackageFiles {
 	return *p.files
 }
 
+// Contents returns cached package contents
+func (p *Package) Contents(packagePool aptly.PackagePool) []string {
+	if p.IsSource {
+		return nil
+	}
+
+	return p.collection.loadContents(p, packagePool)
+}
+
+// CalculateContents looks up contents in package file
+func (p *Package) CalculateContents(packagePool aptly.PackagePool) []string {
+	if p.IsSource {
+		return nil
+	}
+
+	file := p.Files()[0]
+	path, err := packagePool.Path(file.Filename, file.Checksums.MD5)
+	if err != nil {
+		panic(err)
+	}
+
+	contents, err := GetContentsFromDeb(path)
+	if err != nil {
+		panic(err)
+	}
+
+	return contents
+}
+
 // UpdateFiles saves new state of files
 func (p *Package) UpdateFiles(files PackageFiles) {
 	p.files = &files
@@ -432,7 +453,7 @@ func (p *Package) Stanza() (result Stanza) {
 	}
 
 	if p.IsSource {
-		md5, sha1, sha256 := make([]string, 0), make([]string, 0), make([]string, 0)
+		md5, sha1, sha256, sha512 := []string{}, []string{}, []string{}, []string{}
 
 		for _, f := range p.Files() {
 			if f.Checksums.MD5 != "" {
@@ -444,11 +465,21 @@ func (p *Package) Stanza() (result Stanza) {
 			if f.Checksums.SHA256 != "" {
 				sha256 = append(sha256, fmt.Sprintf(" %s %d %s\n", f.Checksums.SHA256, f.Checksums.Size, f.Filename))
 			}
+			if f.Checksums.SHA512 != "" {
+				sha512 = append(sha512, fmt.Sprintf(" %s %d %s\n", f.Checksums.SHA512, f.Checksums.Size, f.Filename))
+			}
 		}
 
 		result["Files"] = strings.Join(md5, "")
-		result["Checksums-Sha1"] = strings.Join(sha1, "")
-		result["Checksums-Sha256"] = strings.Join(sha256, "")
+		if len(sha1) > 0 {
+			result["Checksums-Sha1"] = strings.Join(sha1, "")
+		}
+		if len(sha256) > 0 {
+			result["Checksums-Sha256"] = strings.Join(sha256, "")
+		}
+		if len(sha512) > 0 {
+			result["Checksums-Sha512"] = strings.Join(sha512, "")
+		}
 	} else {
 		f := p.Files()[0]
 		result["Filename"] = f.DownloadURL()
@@ -456,10 +487,13 @@ func (p *Package) Stanza() (result Stanza) {
 			result["MD5sum"] = f.Checksums.MD5
 		}
 		if f.Checksums.SHA1 != "" {
-			result["SHA1"] = " " + f.Checksums.SHA1
+			result["SHA1"] = f.Checksums.SHA1
 		}
 		if f.Checksums.SHA256 != "" {
-			result["SHA256"] = " " + f.Checksums.SHA256
+			result["SHA256"] = f.Checksums.SHA256
+		}
+		if f.Checksums.SHA512 != "" {
+			result["SHA512"] = f.Checksums.SHA512
 		}
 		result["Size"] = fmt.Sprintf("%d", f.Checksums.Size)
 	}

deb/package_collection.go

@@ -3,16 +3,17 @@ package deb
 import (
 	"bytes"
 	"fmt"
+	"path/filepath"
+
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/ugorji/go/codec"
-	"path/filepath"
 )
 
 // PackageCollection does management of packages in DB
 type PackageCollection struct {
-	db           database.Storage
-	encodeBuffer bytes.Buffer
-	codecHandle  *codec.MsgpackHandle
+	db          database.Storage
+	codecHandle *codec.MsgpackHandle
 }
 
 // Verify interface
@@ -161,45 +162,82 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 	return files
 }
 
+// loadContents loads or calculates and saves package contents
+func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.PackagePool) []string {
+	encoded, err := collection.db.Get(p.Key("xC"))
+	if err == nil {
+		contents := []string{}
+
+		decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
+		err = decoder.Decode(&contents)
+		if err != nil {
+			panic("unable to decode contents")
+		}
+
+		return contents
+	}
+
+	if err != database.ErrNotFound {
+		panic("unable to load contents")
+	}
+
+	contents := p.CalculateContents(packagePool)
+
+	var buf bytes.Buffer
+	err = codec.NewEncoder(&buf, collection.codecHandle).Encode(contents)
+	if err != nil {
+		panic("unable to encode contents")
+	}
+
+	err = collection.db.Put(p.Key("xC"), buf.Bytes())
+	if err != nil {
+		panic("unable to save contents")
+	}
+
+	return contents
+}
+
 // Update adds or updates information about package in DB checking for conficts first
 func (collection *PackageCollection) Update(p *Package) error {
-	encoder := codec.NewEncoder(&collection.encodeBuffer, collection.codecHandle)
+	var encodeBuffer bytes.Buffer
 
-	collection.encodeBuffer.Reset()
-	collection.encodeBuffer.WriteByte(0xc1)
-	collection.encodeBuffer.WriteByte(0x1)
+	encoder := codec.NewEncoder(&encodeBuffer, collection.codecHandle)
+
+	encodeBuffer.Reset()
+	encodeBuffer.WriteByte(0xc1)
+	encodeBuffer.WriteByte(0x1)
 	err := encoder.Encode(p)
 	if err != nil {
 		return err
 	}
 
-	err = collection.db.Put(p.Key(""), collection.encodeBuffer.Bytes())
+	err = collection.db.Put(p.Key(""), encodeBuffer.Bytes())
 	if err != nil {
 		return err
 	}
 
 	// Encode offloaded fields one by one
 	if p.files != nil {
-		collection.encodeBuffer.Reset()
+		encodeBuffer.Reset()
 		err = encoder.Encode(*p.files)
 		if err != nil {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xF"), collection.encodeBuffer.Bytes())
+		err = collection.db.Put(p.Key("xF"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
 	}
 
 	if p.deps != nil {
-		collection.encodeBuffer.Reset()
+		encodeBuffer.Reset()
 		err = encoder.Encode(*p.deps)
 		if err != nil {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xD"), collection.encodeBuffer.Bytes())
+		err = collection.db.Put(p.Key("xD"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
@@ -208,13 +246,13 @@ func (collection *PackageCollection) Update(p *Package) error {
 	}
 
 	if p.extra != nil {
-		collection.encodeBuffer.Reset()
+		encodeBuffer.Reset()
 		err = encoder.Encode(*p.extra)
 		if err != nil {
 			return err
 		}
 
-		err = collection.db.Put(p.Key("xE"), collection.encodeBuffer.Bytes())
+		err = collection.db.Put(p.Key("xE"), encodeBuffer.Bytes())
 		if err != nil {
 			return err
 		}
@@ -245,7 +283,7 @@ func (collection *PackageCollection) DeleteByKey(key []byte) error {
 
 // Scan does full scan on all the packages
 func (collection *PackageCollection) Scan(q PackageQuery) (result *PackageList) {
-	result = NewPackageList()
+	result = NewPackageListWithDuplicates(true, 0)
 
 	for _, key := range collection.db.KeysByPrefix([]byte("P")) {
 		pkg, err := collection.ByKey(key)

deb/package_collection_test.go

@@ -64,9 +64,9 @@ func (s *PackageCollectionSuite) TestByKey(c *C) {
 	c.Check(p2.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
 }
 
-func (s *PackageCollectionSuite) TestByKeyOld_0_3(c *C) {
+func (s *PackageCollectionSuite) TestByKeyOld0_3(c *C) {
 	key := []byte("Pi386 vmware-view-open-client 4.5.0-297975+dfsg-4+b1")
-	s.db.Put(key, old_0_3_Package)
+	s.db.Put(key, old0_3Package)
 
 	p, err := s.collection.ByKey(key)
 	c.Check(err, IsNil)
@@ -133,7 +133,7 @@ func (s *PackageCollectionSuite) TestDeleteByKey(c *C) {
 }
 
 // This is old package (pre-0.4) that would habe to be converted
-var old_0_3_Package = []byte{0x8f, 0xac, 0x41, 0x72, 0x63, 0x68, 0x69, 0x74, 0x65, 0x63, 0x74, 0x75, 0x72, 0x65, 0xa4, 0x69, 0x33, 0x38, 0x36,
+var old0_3Package = []byte{0x8f, 0xac, 0x41, 0x72, 0x63, 0x68, 0x69, 0x74, 0x65, 0x63, 0x74, 0x75, 0x72, 0x65, 0xa4, 0x69, 0x33, 0x38, 0x36,
 	0xac, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x73, 0xc0, 0xb1, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65,
 	0x70, 0x65, 0x6e, 0x64, 0x73, 0x49, 0x6e, 0x44, 0x65, 0x70, 0xc0, 0xa7, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x73, 0xdc, 0x0, 0x12,
 	0xb7, 0x6c, 0x69, 0x62, 0x61, 0x74, 0x6b, 0x31, 0x2e, 0x30, 0x2d, 0x30, 0x20, 0x28, 0x3e, 0x3d, 0x20, 0x31, 0x2e, 0x31, 0x32, 0x2e,

deb/package_deps.go

@@ -22,6 +22,12 @@ func parseDependencies(input Stanza, key string) []string {
 
 	delete(input, key)
 
+	value = strings.TrimSpace(value)
+	if value == "" {
+		// empty line is no depdencies
+		return nil
+	}
+
 	result := strings.Split(value, ",")
 	for i := range result {
 		result[i] = strings.TrimSpace(result[i])

deb/package_files.go

@@ -2,12 +2,16 @@ package deb
 
 import (
 	"encoding/binary"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/utils"
+	"fmt"
 	"hash/fnv"
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
 )
 
 // PackageFile is a single file entry in package
@@ -76,3 +80,71 @@ func (files PackageFiles) Swap(i, j int) {
 func (files PackageFiles) Less(i, j int) bool {
 	return files[i].Filename < files[j].Filename
 }
+
+func (files PackageFiles) parseSumField(input string, setter func(sum *utils.ChecksumInfo, data string)) (PackageFiles, error) {
+	for _, line := range strings.Split(input, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		parts := strings.Fields(line)
+
+		if len(parts) < 3 {
+			return nil, fmt.Errorf("unparseable hash sum line: %#v", line)
+		}
+
+		size, err := strconv.ParseInt(parts[1], 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("unable to parse size: %s", err)
+		}
+
+		filename := filepath.Base(parts[len(parts)-1])
+
+		found := false
+		pos := 0
+		for i, file := range files {
+			if file.Filename == filename {
+				found = true
+				pos = i
+				break
+			}
+		}
+
+		if !found {
+			files = append(files, PackageFile{Filename: filename})
+			pos = len(files) - 1
+		}
+
+		files[pos].Checksums.Size = size
+		setter(&files[pos].Checksums, parts[0])
+	}
+
+	return files, nil
+}
+
+// ParseSumFields populates PackageFiles by parsing stanza checksums fields
+func (files PackageFiles) ParseSumFields(stanza Stanza) (PackageFiles, error) {
+	var err error
+
+	files, err = files.parseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	files, err = files.parseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	files, err = files.parseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	files, err = files.parseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	return files, nil
+}

deb/package_files_test.go

@@ -1,11 +1,12 @@
 package deb
 
 import (
-	"github.com/smira/aptly/files"
-	"github.com/smira/aptly/utils"
 	"os"
 	"path/filepath"
 
+	"github.com/smira/aptly/files"
+	"github.com/smira/aptly/utils"
+
 	. "gopkg.in/check.v1"
 )
 

deb/package_test.go

@@ -2,12 +2,13 @@ package deb
 
 import (
 	"bytes"
-	"github.com/smira/aptly/files"
-	"github.com/smira/aptly/utils"
 	"os"
 	"path/filepath"
 	"regexp"
 
+	"github.com/smira/aptly/files"
+	"github.com/smira/aptly/utils"
+
 	. "gopkg.in/check.v1"
 )
 
@@ -22,7 +23,7 @@ func (s *PackageSuite) SetUpTest(c *C) {
 	s.stanza = packageStanza.Copy()
 
 	buf := bytes.NewBufferString(sourcePackageMeta)
-	s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza()
+	s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza(false)
 }
 
 func (s *PackageSuite) TestNewFromPara(c *C) {
@@ -43,7 +44,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 }
 
 func (s *PackageSuite) TestNewUdebFromPara(c *C) {
-	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
 	p := NewUdebPackageFromControlFile(stanza)
 
 	c.Check(p.IsSource, Equals, false)
@@ -125,6 +126,10 @@ func (s *PackageSuite) TestStanza(c *C) {
 	p := NewPackageFromControlFile(s.stanza.Copy())
 	stanza := p.Stanza()
 
+	for k := range s.stanza {
+		c.Check(stanza[k], Equals, s.stanza[k])
+	}
+
 	c.Assert(stanza, DeepEquals, s.stanza)
 
 	p, _ = NewSourcePackageFromControlFile(s.sourceStanza.Copy())
@@ -152,7 +157,7 @@ func (s *PackageSuite) TestGetField(c *C) {
 
 	p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
 
-	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
 	p5 := NewUdebPackageFromControlFile(stanza5)
 
 	c.Check(p.GetField("$Source"), Equals, "alien-arena")
@@ -445,7 +450,7 @@ func (s *PackageSuite) TestVerifyFiles(c *C) {
 	c.Check(result, Equals, true)
 }
 
-var packageStanza = Stanza{"Source": "alien-arena", "Pre-Depends": "dpkg (>= 1.6)", "Suggests": "alien-arena-mars", "Recommends": "aliean-arena-luna", "Depends": "libc6 (>= 2.7), alien-arena-data (>= 7.40)", "Filename": "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", "SHA1": " 46955e48cad27410a83740a21d766ce362364024", "SHA256": " eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", "Priority": "extra", "Maintainer": "Debian Games Team <pkg-games-devel@lists.alioth.debian.org>", "Description": "Common files for Alien Arena client and server ALIEN ARENA is a standalone 3D first person online deathmatch shooter\n crafted from the original source code of Quake II and Quake III, released\n by id Software under the GPL license. With features including 32 bit\n graphics, new particle engine and effects, light blooms, reflective water,\n hi resolution textures and skins, hi poly models, stain maps, ALIEN ARENA\n pushes the envelope of graphical beauty rivaling today's top games.\n .\n This package installs the common files for Alien Arena.\n", "Homepage": "http://red.planetarena.org", "Tag": "role::app-data, role::shared-lib, special::auto-inst-parts", "Installed-Size": "456", "Version": "7.40-2", "Replaces": "alien-arena (<< 7.33-1)", "Size": "187518", "MD5sum": "1e8cba92c41420aa7baa8a5718d67122", "Package": "alien-arena-common", "Section": "contrib/games", "Architecture": "i386"}
+var packageStanza = Stanza{"Source": "alien-arena", "Pre-Depends": "dpkg (>= 1.6)", "Suggests": "alien-arena-mars", "Recommends": "aliean-arena-luna", "Depends": "libc6 (>= 2.7), alien-arena-data (>= 7.40)", "Filename": "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", "SHA1": "46955e48cad27410a83740a21d766ce362364024", "SHA256": "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", "Priority": "extra", "Maintainer": "Debian Games Team <pkg-games-devel@lists.alioth.debian.org>", "Description": "Common files for Alien Arena client and server ALIEN ARENA is a standalone 3D first person online deathmatch shooter\n crafted from the original source code of Quake II and Quake III, released\n by id Software under the GPL license. With features including 32 bit\n graphics, new particle engine and effects, light blooms, reflective water,\n hi resolution textures and skins, hi poly models, stain maps, ALIEN ARENA\n pushes the envelope of graphical beauty rivaling today's top games.\n .\n This package installs the common files for Alien Arena.\n", "Homepage": "http://red.planetarena.org", "Tag": "role::app-data, role::shared-lib, special::auto-inst-parts", "Installed-Size": "456", "Version": "7.40-2", "Replaces": "alien-arena (<< 7.33-1)", "Size": "187518", "MD5sum": "1e8cba92c41420aa7baa8a5718d67122", "Package": "alien-arena-common", "Section": "contrib/games", "Architecture": "i386"}
 
 const sourcePackageMeta = `Package: access-modifier-checker
 Binary: libaccess-modifier-checker-java, libaccess-modifier-checker-java-doc

deb/ppa.go

@@ -2,10 +2,11 @@ package deb
 
 import (
 	"fmt"
-	"github.com/smira/aptly/utils"
 	"os/exec"
 	"regexp"
 	"strings"
+
+	"github.com/smira/aptly/utils"
 )
 
 var ppaRegexp = regexp.MustCompile("^ppa:([^/]+)/(.+)$")